You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/12/29 21:59:58 UTC

[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 718 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/718/

1 tests failed.
FAILED:  org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([F2B4D7F138C297F6:337C0AB799A4465F]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:539)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability(TestLBHttpSolrServer.java:223)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55275/solr
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:492)
	... 43 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:463)
	... 46 more




Build Log:
[...truncated 11493 lines...]
   [junit4] Suite: org.apache.solr.client.solrj.TestLBHttpSolrServer
   [junit4]   2> log4j:WARN No such property [conversionPattern] in org.apache.solr.util.SolrLogLayout.
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/init-core-data-001
   [junit4]   2> 4689 T11 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 4946 T11 oas.SolrTestCaseJ4.setUp ###Starting testReliability
   [junit4]   2> 6530 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 6984 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18943
   [junit4]   2> 7181 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 7378 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 7379 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001
   [junit4]   2> 7380 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/'
   [junit4]   2> 8746 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/solr.xml
   [junit4]   2> 9236 T11 oasc.CoreContainer.<init> New CoreContainer 1937261928
   [junit4]   2> 9246 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/]
   [junit4]   2> 9591 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 9593 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 9594 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 9595 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 9596 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 9601 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 9602 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 9603 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 9604 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 9606 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 10231 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 10270 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 10281 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 10300 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 11279 T23 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/'
   [junit4]   2> 11632 T23 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 11924 T23 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 11987 T23 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf/schema.xml
   [junit4]   2> 12044 T23 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 12558 T23 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 12587 T23 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/
   [junit4]   2> 13078 T23 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 13170 T23 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/
   [junit4]   2> 13172 T23 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 13410 T23 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data
   [junit4]   2> 13414 T23 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index/
   [junit4]   2> 13417 T23 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 13456 T23 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index
   [junit4]   2> 13858 T23 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 13866 T23 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 16208 T23 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 16429 T23 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16430 T23 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16431 T23 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 16434 T23 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 16436 T23 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 16445 T23 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 16448 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 16451 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 16453 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 16454 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 16455 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 16457 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 16458 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 16466 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 16468 T23 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 16504 T23 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 16505 T23 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 16506 T23 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 16515 T23 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 16559 T23 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 16665 T23 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 16933 T23 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16975 T23 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16978 T23 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 16997 T23 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 17155 T23 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 17156 T23 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 17267 T23 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 17268 T23 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 17559 T23 oass.SolrIndexSearcher.<init> Opening Searcher@28f7852b[collection1] main
   [junit4]   2> 17644 T23 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf
   [junit4]   2> 17645 T23 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf}
   [junit4]   2> 17834 T23 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf
   [junit4]   2> 17836 T23 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 18105 T23 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 18106 T23 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 18119 T24 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28f7852b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 18127 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1
   [junit4]   2> 18128 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@1be46a9d
   [junit4]   2> 21999 T18 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 1891
   [junit4]   2> 22420 T18 C0 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 23973 T18 C0 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_2,generation=2}
   [junit4]   2> 23978 T18 C0 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 24330 T18 C0 oass.SolrIndexSearcher.<init> Opening Searcher@29409deb[collection1] main
   [junit4]   2> 24347 T18 C0 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 24357 T24 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29409deb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10)))}
   [junit4]   2> 24359 T18 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&version=2&wt=javabin&waitSearcher=true&softCommit=false} {commit=} 0 1939
   [junit4]   2> 24384 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 24387 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59518
   [junit4]   2> 24388 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 24389 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 24390 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001
   [junit4]   2> 24391 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/'
   [junit4]   2> 24468 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/solr.xml
   [junit4]   2> 24601 T11 oasc.CoreContainer.<init> New CoreContainer 1179354676
   [junit4]   2> 24602 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/]
   [junit4]   2> 24605 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 24606 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 24607 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 24608 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 24608 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 24609 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 24610 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 24611 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 24612 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 24613 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 24614 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 24615 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 24616 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 24617 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 24626 T35 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/'
   [junit4]   2> 24694 T35 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 24776 T35 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 24778 T35 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/conf/schema.xml
   [junit4]   2> 24788 T35 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 24813 T35 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 24815 T35 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/
   [junit4]   2> 24816 T35 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 24816 T35 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/
   [junit4]   2> 24817 T35 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 24819 T35 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data
   [junit4]   2> 24821 T35 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index/
   [junit4]   2> 24822 T35 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 24824 T35 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index
   [junit4]   2> 24832 T35 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 24833 T35 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 24836 T35 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 24837 T35 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 24837 T35 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 24838 T35 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 24838 T35 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 24839 T35 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 24840 T35 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 24840 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 24841 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 24842 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 24842 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 24843 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 24844 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 24844 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 24845 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 24845 T35 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 24846 T35 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 24846 T35 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 24847 T35 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 24847 T35 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 24848 T35 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 24848 T35 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 24875 T35 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24878 T35 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24880 T35 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24883 T35 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24886 T35 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 24886 T35 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 24891 T35 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 24891 T35 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 24892 T35 oass.SolrIndexSearcher.<init> Opening Searcher@4352c50f[collection1] main
   [junit4]   2> 24893 T35 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/conf
   [junit4]   2> 24893 T35 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/conf}
   [junit4]   2> 24894 T35 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/conf
   [junit4]   2> 24895 T35 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 24896 T35 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24897 T36 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4352c50f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 24897 T35 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24898 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1
   [junit4]   2> 24899 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@4a669e59
   [junit4]   2> 24909 T31 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 6
   [junit4]   2> 24915 T31 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 24951 T31 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_2,generation=2}
   [junit4]   2> 24952 T31 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 24957 T31 C1 oass.SolrIndexSearcher.<init> Opening Searcher@197ef254[collection1] main
   [junit4]   2> 24957 T31 C1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 24958 T36 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@197ef254[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10)))}
   [junit4]   2> 24958 T31 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&version=2&wt=javabin&waitSearcher=true&softCommit=false} {commit=} 0 44
   [junit4]   2> 24970 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 24973 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55275
   [junit4]   2> 24974 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 24974 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 24975 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001
   [junit4]   2> 24975 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/'
   [junit4]   2> 24991 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/solr.xml
   [junit4]   2> 25057 T11 oasc.CoreContainer.<init> New CoreContainer 688072353
   [junit4]   2> 25058 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/]
   [junit4]   2> 25060 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 25060 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 25061 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 25061 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 25062 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 25062 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 25063 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 25063 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 25064 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 25064 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 25065 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 25066 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 25067 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 25067 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 25073 T47 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/'
   [junit4]   2> 25149 T47 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 25221 T47 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 25223 T47 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/conf/schema.xml
   [junit4]   2> 25237 T47 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 25261 T47 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 25263 T47 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/
   [junit4]   2> 25264 T47 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 25265 T47 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/
   [junit4]   2> 25266 T47 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 25268 T47 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data
   [junit4]   2> 25270 T47 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index/
   [junit4]   2> 25272 T47 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 25274 T47 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index
   [junit4]   2> 25284 T47 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 25284 T47 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25288 T47 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 25290 T47 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25290 T47 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25291 T47 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25292 T47 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 25293 T47 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 25294 T47 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 25295 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 25296 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 25297 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 25298 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 25299 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 25299 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 25300 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 25301 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 25302 T47 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 25302 T47 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 25303 T47 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 25304 T47 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 25304 T47 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 25305 T47 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 25306 T47 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 25309 T47 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25312 T47 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25315 T47 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25318 T47 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25322 T47 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 25323 T47 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 25329 T47 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 25329 T47 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25330 T47 oass.SolrIndexSearcher.<init> Opening Searcher@416ed704[collection1] main
   [junit4]   2> 25332 T47 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/conf
   [junit4]   2> 25332 T47 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/conf}
   [junit4]   2> 25333 T47 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/conf
   [junit4]   2> 25335 T47 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25336 T47 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25336 T48 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@416ed704[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25337 T47 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 25338 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1
   [junit4]   2> 25339 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@5bc55fe5
   [junit4]   2> 25354 T42 C2 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 10
   [junit4]   2> 25360 T42 C2 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 25402 T42 C2 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_2,generation=2}
   [junit4]   2> 25403 T42 C2 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 25410 T42 C2 oass.SolrIndexSearcher.<init> Opening Searcher@10e156cd[collection1] main
   [junit4]   2> 25411 T42 C2 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 25411 T48 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@10e156cd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10)))}
   [junit4]   2> 25412 T42 C2 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&version=2&wt=javabin&waitSearcher=true&softCommit=false} {commit=} 0 52
   [junit4]   2> 25416 T11 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1179354676
   [junit4]   2> 25417 T11 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4a669e59
   [junit4]   2> 25418 T11 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 25420 T11 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 25420 T11 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 25421 T11 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 25426 T11 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 25428 T11 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 25429 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index;done=false>>]
   [junit4]   2> 25430 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index
   [junit4]   2> 25760 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data;done=false>>]
   [junit4]   2> 25761 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data
   [junit4]   2> 25765 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 26659 T11 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1937261928
   [junit4]   2> 26659 T50 oahic.DefaultRequestDirector.tryExecute I/O exception (java.net.SocketException) caught when processing request: Connection reset
   [junit4]   2> 26661 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 26663 T50 oahic.DefaultRequestDirector.tryExecute Retrying request
   [junit4]   2> 26680 T40 C2 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={version=2&wt=javabin&q=*:*} hits=10 status=0 QTime=272 
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@1be46a9d
   [junit4]   2> 26680 T22 C3 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={version=2&wt=javabin&q=*:*} hits=10 status=0 QTime=793 
   [junit4]   2> 26748 T22 C3 oass.SolrDispatchFilter.writeResponse Unable to write response, client closed connection or we are shutting down org.eclipse.jetty.io.EofException
   [junit4]   2> 	at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 	at org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:214)
   [junit4]   2> 	at org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:207)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:98)
   [junit4]   2> 	at org.apache.solr.response.BinaryResponseWriter.write(BinaryResponseWriter.java:52)
   [junit4]   2> 	at org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:50)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:765)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:422)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 26845 T16 C3 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={version=2&sort=_docid_+asc&wt=javabin&distrib=false&q=*:*&rows=0} hits=10 status=0 QTime=187 
   [junit4]   2> 26846 T16 C3 oass.SolrDispatchFilter.writeResponse Unable to write response, client closed connection or we are shutting down org.eclipse.jetty.io.EofException
   [junit4]   2> 	at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:142)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpOutput.write(HttpOutput.java:107)
   [junit4]   2> 	at org.apache.solr.common.util.FastOutputStream.flush(FastOutputStream.java:214)
   [junit4]   2> 	at org.apache.solr.common.util.FastOutputStream.flushBuffer(FastOutputStream.java:207)
   [junit4]   2> 	at org.apache.solr.common.util.JavaBinCodec.marshal(JavaBinCodec.java:98)
   [junit4]   2> 	at org.apache.solr.response.BinaryResponseWriter.write(BinaryResponseWriter.java:52)
   [junit4]   2> 	at org.apache.solr.response.QueryResponseWriterUtil.writeQueryResponse(QueryResponseWriterUtil.java:50)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.writeResponse(SolrDispatchFilter.java:765)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:422)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]   2> 	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]   2> 	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]   2> 	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 26848 T16 C3 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1be46a9d
   [junit4]   2> 26849 T16 C3 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 26850 T16 C3 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 26850 T16 C3 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 26851 T16 C3 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 26858 T16 C3 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 26860 T16 C3 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 26861 T16 C3 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index;done=false>>]
   [junit4]   2> 26861 T16 C3 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index
   [junit4]   2> 26927 T16 C3 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data;done=false>>]
   [junit4]   2> 26928 T16 C3 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data
   [junit4]   2> 26964 T11 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=688072353
   [junit4]   2> 26965 T11 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5bc55fe5
   [junit4]   2> 26966 T11 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 26967 T11 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 26968 T11 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 26969 T11 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 26976 T11 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 26977 T11 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 26978 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data;done=false>>]
   [junit4]   2> 26979 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data
   [junit4]   2> 26981 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index;done=false>>]
   [junit4]   2> 26981 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index
   [junit4]   2> 27047 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 27193 T11 oas.SolrTestCaseJ4.tearDown ###Ending testReliability
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLBHttpSolrServer -Dtests.method=testReliability -Dtests.seed=F2B4D7F138C297F6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=sq_AL -Dtests.timezone=America/Guyana -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   22.7s J1 | TestLBHttpSolrServer.testReliability <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F2B4D7F138C297F6:337C0AB799A4465F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:539)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.client.solrj.TestLBHttpSolrServer.testReliability(TestLBHttpSolrServer.java:223)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55275/solr
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:576)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:492)
   [junit4]    > 	... 43 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:463)
   [junit4]    > 	... 46 more
   [junit4]   2> 27581 T11 oas.SolrTestCaseJ4.setUp ###Starting testTwoServers
   [junit4]   2> 27599 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 27603 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41594
   [junit4]   2> 27604 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 27605 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 27606 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001
   [junit4]   2> 27606 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/'
   [junit4]   2> 27687 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/solr.xml
   [junit4]   2> 27851 T11 oasc.CoreContainer.<init> New CoreContainer 1124745054
   [junit4]   2> 27851 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/]
   [junit4]   2> 27854 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 27855 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 27855 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 27856 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 27857 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 27858 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 27858 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 27859 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 27860 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 27860 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 27862 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 27863 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 27863 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 27864 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 27873 T60 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/'
   [junit4]   2> 27966 T60 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 28052 T60 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 28054 T60 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf/schema.xml
   [junit4]   2> 28074 T60 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 28104 T60 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 28106 T60 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/
   [junit4]   2> 28107 T60 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockFSDirectoryFactory
   [junit4]   2> 28108 T60 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/
   [junit4]   2> 28108 T60 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 28111 T60 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data
   [junit4]   2> 28113 T60 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index/
   [junit4]   2> 28114 T60 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 28116 T60 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index
   [junit4]   2> 28127 T60 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 28128 T60 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 28131 T60 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 28132 T60 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 28133 T60 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 28134 T60 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 28134 T60 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 28135 T60 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 28136 T60 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 28136 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 28138 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 28138 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 28139 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 28140 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 28140 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 28141 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 28142 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: org.apache.solr.handler.admin.ShowFileRequestHandler
   [junit4]   2> 28143 T60 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 28143 T60 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 28144 T60 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 28144 T60 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 28145 T60 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 28146 T60 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 28147 T60 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 28174 T60 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28177 T60 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28180 T60 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28184 T60 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28188 T60 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 28189 T60 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 28194 T60 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 28195 T60 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 28195 T60 oass.SolrIndexSearcher.<init> Opening Searcher@5fdbf01e[collection1] main
   [junit4]   2> 28197 T60 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf
   [junit4]   2> 28197 T60 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf}
   [junit4]   2> 28198 T60 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/conf
   [junit4]   2> 28200 T60 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 28200 T60 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 28201 T61 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5fdbf01e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 28201 T60 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 28203 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1
   [junit4]   2> 28203 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@7ed34a97
   [junit4]   2> 28217 T57 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={version=2&wt=javabin} {add=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]} 0 9
   [junit4]   2> 28222 T57 C4 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 28274 T57 C4 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index lockFactory=org.apache.lucene.store.NativeFSLockFactory@b85c3e6),segFN=segments_2,generation=2}
   [junit4]   2> 28275 T57 C4 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 28281 T57 C4 oass.SolrIndexSearcher.<init> Opening Searcher@2d65e4c4[collection1] main
   [junit4]   2> 28282 T57 C4 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28282 T61 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2d65e4c4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c10)))}
   [junit4]   2> 28283 T57 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={commit=true&version=2&wt=javabin&waitSearcher=true&softCommit=false} {commit=} 0 62
   [junit4]   2> 28306 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 28309 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19917
   [junit4]   2> 28310 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 28311 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 28312 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001
   [junit4]   2> 28312 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/'
   [junit4]   2> 28378 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/solr.xml
   [junit4]   2> 28518 T11 oasc.CoreContainer.<init> New CoreContainer 1010831945
   [junit4]   2> 28519 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/]
   [junit4]   2> 28521 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 28522 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 28523 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 28523 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 28524 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 28525 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 28525 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 28526 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 28527 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 28527 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 28528 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 28529 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 28530 T11 oasl.LogWatcher.newRegisteredL

[...truncated too long message...]

T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index;done=false>>]
   [junit4]   2> 37542 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data/index
   [junit4]   2> 37602 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data;done=false>>]
   [junit4]   2> 37603 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-0-001/collection1/data
   [junit4]   2> 37604 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 37740 T11 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=685957714
   [junit4]   2> 37741 T11 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5c834ea7
   [junit4]   2> 37742 T11 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 37742 T11 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 37743 T11 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 37744 T11 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 37748 T11 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 37750 T11 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 37751 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index;done=false>>]
   [junit4]   2> 37751 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data/index
   [junit4]   2> 37765 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data;done=false>>]
   [junit4]   2> 37766 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-1-001/collection1/data
   [junit4]   2> 37768 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 37950 T11 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=623152470
   [junit4]   2> 37951 T11 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@51205cb7
   [junit4]   2> 37952 T11 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=10,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 37953 T11 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 37954 T11 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 37954 T11 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 37958 T11 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 37960 T11 oasc.CachingDirectoryFactory.close Closing MockFSDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 37960 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data;done=false>>]
   [junit4]   2> 37961 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data
   [junit4]   2> 37963 T11 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index;done=false>>]
   [junit4]   2> 37964 T11 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001/instance-2-001/collection1/data/index
   [junit4]   2> 38024 T11 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 38164 T11 oas.SolrTestCaseJ4.tearDown ###Ending testSimple
   [junit4]   2> 38170 T11 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.TestLBHttpSolrServer-F2B4D7F138C297F6-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=5, maxDocsPerChunk=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=5)), sim=DefaultSimilarity, locale=sq_AL, timezone=America/Guyana
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=3,free=151375352,total=311427072
   [junit4]   2> NOTE: All tests run in this JVM: [TestLBHttpSolrServer]
   [junit4] Completed on J1 in 39.13s, 3 tests, 1 error <<< FAILURES!

[...truncated 147 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:203: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 52 suites, 329 tests, 1 error

Total time: 185 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 10181433 bytes
Compression is 0.0%
Took 2.6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 764 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/764/

6 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:64327/_lsi/kv/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:64327/_lsi/kv/collection1
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:538)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:568)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:547)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19929/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19929/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19929/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52857/l_dfc/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52857/l_dfc/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52857/l_dfc/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37270/e/xn/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37270/e/xn/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64263/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64263/collection1
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64263/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19612/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19612/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19612/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9857 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/init-core-data-001
   [junit4]   2> 39678 T44 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 39722 T44 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 39733 T45 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 40034 T44 oasc.ZkTestServer.run start zk server on port:42600
   [junit4]   2> 40168 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40490 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40678 T52 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a776dd4 name:ZooKeeperConnection Watcher:127.0.0.1:42600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40679 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40686 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40701 T44 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 40831 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40832 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40835 T55 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b316c11 name:ZooKeeperConnection Watcher:127.0.0.1:42600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40835 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40836 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40836 T44 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 40849 T44 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 40855 T44 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 40859 T44 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 40865 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 40891 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 40899 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 40900 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 40906 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 40907 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 40913 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 40913 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 40919 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 40920 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 40924 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 40924 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 40929 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 40930 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 40934 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 40934 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 40938 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 40939 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 40944 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 40945 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 40950 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 40950 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 42148 T44 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores/collection1
   [junit4]   2> 42152 T44 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 42155 T44 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64249
   [junit4]   2> 42156 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 42157 T44 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 42157 T44 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001
   [junit4]   2> 42158 T44 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/'
   [junit4]   2> 42194 T44 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/solr.xml
   [junit4]   2> 42242 T44 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores
   [junit4]   2> 42243 T44 oasc.CoreContainer.<init> New CoreContainer 602706895
   [junit4]   2> 42244 T44 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/]
   [junit4]   2> 42245 T44 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 42245 T44 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 42246 T44 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 42246 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 42246 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 42247 T44 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 42247 T44 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 42247 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 42248 T44 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 42248 T44 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 42249 T44 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 42249 T44 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 42250 T44 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 42250 T44 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 42251 T44 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 42251 T44 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42600/solr
   [junit4]   2> 42297 T44 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 42298 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 42299 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 42301 T67 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@344f223 name:ZooKeeperConnection Watcher:127.0.0.1:42600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 42302 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 42303 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 42322 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 42324 T70 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e9d9f17 name:ZooKeeperConnection Watcher:127.0.0.1:42600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 42325 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 42343 T44 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 42361 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 42370 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 42374 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 42377 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 42392 T44 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 42395 T44 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64248_
   [junit4]   2> 42404 T44 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64248_
   [junit4]   2> 42409 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 42412 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 42417 T44 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 42436 T44 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:64248_
   [junit4]   2> 42436 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 42439 T44 oasc.Overseer.start Overseer (id=93325003265212419-127.0.0.1:64248_-n_0000000000) starting
   [junit4]   2> 42460 T44 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 42494 T44 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 42511 T72 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 42512 T44 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 42514 T44 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 42517 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 42545 T71 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 42545 T44 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores
   [junit4]   2> 42562 T44 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores/collection1/
   [junit4]   2> 42563 T44 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 42564 T74 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 42565 T74 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 42578 T74 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 42578 T70 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 42586 T71 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64248_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64248",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 42621 T71 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64248_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64248",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 42629 T71 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 42647 T71 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 42667 T70 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 43579 T74 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 43579 T74 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 43581 T74 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 43581 T74 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 43583 T74 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 43584 T74 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores/collection1/'
   [junit4]   2> 43644 T74 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 43672 T74 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 43715 T74 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 43753 T74 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 43821 T74 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 43824 T74 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 43882 T74 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 44705 T74 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 44739 T74 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 44744 T74 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 44823 T74 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 44860 T74 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 44868 T74 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 44880 T74 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 44880 T74 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 44881 T74 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 44883 T74 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 44883 T74 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 44907 T74 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 44908 T74 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 44908 T74 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/tempDir-001/control/data/
   [junit4]   2> 44918 T74 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a49ed1a
   [junit4]   2> 45085 T74 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/tempDir-001/control/data
   [junit4]   2> 45086 T74 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/tempDir-001/control/data/index/
   [junit4]   2> 45086 T74 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 45087 T74 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/tempDir-001/control/data/index
   [junit4]   2> 45088 T74 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=1.6708984375, floorSegmentMB=1.7021484375, forceMergeDeletesPctAllowed=21.005758817213966, segmentsPerTier=29.0, maxCFSSegmentSizeMB=1.4853515625, noCFSRatio=0.5632543914757252
   [junit4]   2> 45089 T74 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@71297c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5aa1ea7d),segFN=segments_1,generation=1}
   [junit4]   2> 45090 T74 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 45101 T74 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 45195 T74 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 45195 T74 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 45196 T74 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 45196 T74 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 45197 T74 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 45198 T74 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 45199 T74 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 45199 T74 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 45210 T74 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 45212 T74 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 45213 T74 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 45215 T74 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 45216 T74 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 45218 T74 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 45220 T74 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 45221 T74 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 45223 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 45225 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 45227 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 45228 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 45230 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 45231 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 45233 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 45234 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 45236 T74 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 45237 T74 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 45265 T74 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 45269 T74 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 45272 T74 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 45275 T74 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 45286 T74 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 45342 T74 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 45343 T74 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 45345 T74 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.503332652245922]
   [junit4]   2> 45346 T74 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@71297c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5aa1ea7d),segFN=segments_1,generation=1}
   [junit4]   2> 45346 T74 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 45347 T74 oass.SolrIndexSearcher.<init> Opening Searcher@7937c9b2[collection1] main
   [junit4]   2> 45348 T74 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 45350 T74 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 45350 T74 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 45363 T74 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 45364 T74 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 45364 T74 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 45366 T74 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 45366 T74 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 45367 T74 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 45367 T74 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 45370 T75 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7937c9b2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 45380 T74 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 45381 T78 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64248 collection:control_collection shard:shard1
   [junit4]   2> 45382 T44 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 45383 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 45415 T78 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 45425 T78 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 45428 T70 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 45429 T78 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 45429 T78 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@12a74704 url=http://127.0.0.1:64248/collection1 node=127.0.0.1:64248_ C3_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:64248_, base_url=http://127.0.0.1:64248, state=down}
   [junit4]   2> 45430 T78 C3 P64248 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64248/collection1/
   [junit4]   2> 45430 T71 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 45430 T78 C3 P64248 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 45431 T78 C3 P64248 oasc.SyncStrategy.syncToMe http://127.0.0.1:64248/collection1/ has no replicas
   [junit4]   2> 45431 T78 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64248/collection1/ shard1
   [junit4]   2> 45446 T78 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 45446 T70 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 45454 T80 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 45456 T71 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64248",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 45487 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 45488 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45490 T83 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@141b9009 name:ZooKeeperConnection Watcher:127.0.0.1:42600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45490 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45491 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 45492 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 45498 T44 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 45563 T83 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 45609 T78 oasc.ZkController.register We are http://127.0.0.1:64248/collection1/ and leader is http://127.0.0.1:64248/collection1/
   [junit4]   2> 45619 T78 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:64248
   [junit4]   2> 45619 T78 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 45620 T78 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 45620 T78 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 45624 T70 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 45629 T71 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64248_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64248",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 45631 T71 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64248_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64248",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 45738 T70 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 45738 T83 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 47305 T44 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/cores/collection1
   [junit4]   2> 47309 T44 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 47310 T44 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 47312 T44 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64264
   [junit4]   2> 47313 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 47313 T44 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 47314 T44 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001
   [junit4]   2> 47314 T44 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/'
   [junit4]   2> 47362 T44 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/solr.xml
   [junit4]   2> 47423 T44 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/cores
   [junit4]   2> 47424 T44 oasc.CoreContainer.<init> New CoreContainer 379270901
   [junit4]   2> 47424 T44 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/]
   [junit4]   2> 47426 T44 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 47426 T44 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 47426 T44 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 47427 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 47427 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 47427 T44 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 47428 T44 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 47428 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 47428 T44 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 47429 T44 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 47429 T44 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 47430 T44 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 47430 T44 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 47431 T44 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 47431 T44 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 47431 T44 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42600/solr
   [junit4]   2> 47432 T44 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 47432 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 47433 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47435 T95 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b838222 name:ZooKeeperConnection Watcher:127.0.0.1:42600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47436 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47437 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 47439 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47441 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71b38f6a name:ZooKeeperConnection Watcher:127.0.0.1:42600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47441 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47449 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 48464 T44 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64263_
   [junit4]   2> 48466 T44 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64263_
   [junit4]   2> 48473 T44 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 48488 T44 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/cores
   [junit4]   2> 48492 T44 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/cores/collection1/
   [junit4]   2> 48492 T44 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 48494 T99 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 48495 T99 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 48496 T70 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48496 T99 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 48498 T71 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64263_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64263",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 48500 T71 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:64263_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64263",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 48500 T71 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 48501 T71 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 48504 T98 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 48504 T83 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 48504 T70 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 49497 T99 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 49498 T99 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 49500 T99 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 49500 T99 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 49503 T99 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49503 T99 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 591C9B8AB1FFE318-001/shard-1-001/cores/collection1/'
   [junit4]   2> 49528 T99 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 49533 T99 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 49561 T99 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 49598 T99 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 49668 T99 oasc.SolrConfig.<init> Loaded SolrCon

[...truncated too long message...]

nit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_lf/leader_elect/shard1/election/93325098139910152-core_node2-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1581942 T7623 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:52870/l_dfc
   [junit4]   2> 1581943 T7623 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:52834/l_dfc
   [junit4]   2> 1581943 T7623 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:52845/l_dfc
   [junit4]   2> 1581943 T7623 oasc.SocketProxy.close WARN Closing 14 connections to: http://127.0.0.1:52857/l_dfc
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test -Dtests.seed=591C9B8AB1FFE318 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_PA -Dtests.timezone=America/Metlakatla -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   94.2s J1 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52857/l_dfc/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([591C9B8AB1FFE318:D148A4501F038EE0]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52857/l_dfc/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 54 more
   [junit4]   2> 1581964 T7623 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 591C9B8AB1FFE318-001
   [junit4]   2> 94215 T7622 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=FSTOrd50, timestamp=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=LuceneFixedGap), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=FSTOrd50, a_t=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=es_PA, timezone=America/Metlakatla
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=36659168,total=455081984
   [junit4]   2> NOTE: All tests run in this JVM: [IndexSchemaTest, MigrateRouteKeyTest, TestExpandComponent, TestMiniSolrCloudCluster, TestPhraseSuggestions, FullSolrCloudDistribCmdsTest, CoreAdminCreateDiscoverTest, SolrInfoMBeanTest, TestSerializedLuceneMatchVersion, AssignTest, TestCollectionAPI, TestRecovery, ZkControllerTest, CursorMarkTest, PingRequestHandlerTest, JSONWriterTest, TestManagedSchemaFieldResource, DeleteShardTest, AtomicUpdatesTest, SimplePostToolTest, OpenCloseCoreStressTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, BufferStoreTest, WordBreakSolrSpellCheckerTest, DistributedQueueTest, TestConfig, AutoCommitTest, TestManagedSchemaDynamicFieldResource, TestStressVersions, CircularListTest, TestBinaryResponseWriter, DirectSolrConnectionTest, DistributedSuggestComponentTest, AnalyticsMergeStrategyTest, TestReplicationHandlerBackup, TestCollapseQParserPlugin, SuggesterTSTTest, BasicZkTest, CoreMergeIndexesAdminHandlerTest, DistributedFacetPivotSmallTest, TestStressLucene, DistributedTermsComponentTest, FieldAnalysisRequestHandlerTest, DistributedIntervalFacetingTest, TestTolerantSearch, PathHierarchyTokenizerFactoryTest, TestConfigReload, VMParamsZkACLAndCredentialsProvidersTest, TestSearcherReuse, TestSolrDeletionPolicy1, FacetPivotSmallTest, UniqFieldsUpdateProcessorFactoryTest, TestZkChroot, DistributedQueryComponentCustomSortTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J1 in 94.50s, 1 test, 1 error <<< FAILURES!

[...truncated 924 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1863 tests, 6 errors, 56 ignored (9 assumptions)

Total time: 164 minutes 13 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 665451869 bytes
Compression is 0.0%
Took 1 min 58 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 763 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/763/

7 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1423929401703,generation=2,filelist=[_5in.fdt, _5in.fdx, _5in.fnm, _5in.nvd, _5in.nvm, _5in.si, _5in_Memory_0.ram, _5jv.fdt, _5jv.fdx, _5jv.fnm, _5jv.nvd, _5jv.nvm, _5jv.si, _5jv_Memory_0.ram, _5jw.cfe, _5jw.cfs, _5jw.si, _5jx.fdt, _5jx.fdx, _5jx.fnm, _5jx.nvd, _5jx.nvm, _5jx.si, _5jx_Memory_0.ram, _5jy.fdt, _5jy.fdx, _5jy.fnm, _5jy.nvd, _5jy.nvm, _5jy.si, _5jy_Memory_0.ram, _5jz.fdt, _5jz.fdx, _5jz.fnm, _5jz.nvd, _5jz.nvm, _5jz.si, _5jz_Memory_0.ram, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0_Memory_0.ram, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1_Memory_0.ram, _5k2.fdt, _5k2.fdx, _5k2.fnm, _5k2.nvd, _5k2.nvm, _5k2.si, _5k2_Memory_0.ram, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3_Memory_0.ram, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4_Memory_0.ram, _5k5.fdt, _5k5.fdx, _5k5.fnm, _5k5.nvd, _5k5.nvm, _5k5.si, _5k5_Memory_0.ram, _5k6.fdt, _5k6.fdx, _5k6.fnm, _5k6.nvd, _5k6.nvm, _5k6.si, _5k6_Memory_0.ram, _5k7.fdt, _5k7.fdx, _5k7.fnm, _5k7.nvd, _5k7.nvm, _5k7.si, _5k7_Memory_0.ram, _5k8.fdt, _5k8.fdx, _5k8.fnm, _5k8.nvd, _5k8.nvm, _5k8.si, _5k8_Memory_0.ram, _5k9.fdt, _5k9.fdx, _5k9.fnm, _5k9.nvd, _5k9.nvm, _5k9.si, _5k9_Memory_0.ram, _5ka.fdt, _5ka.fdx, _5ka.fnm, _5ka.nvd, _5ka.nvm, _5ka.si, _5ka_Memory_0.ram, _5kb.fdt, _5kb.fdx, _5kb.fnm, _5kb.nvd, _5kb.nvm, _5kb.si, _5kb_Memory_0.ram, _5kc.fdt, _5kc.fdx, _5kc.fnm, _5kc.nvd, _5kc.nvm, _5kc.si, _5kc_Memory_0.ram, _5kd.fdt, _5kd.fdx, _5kd.fnm, _5kd.nvd, _5kd.nvm, _5kd.si, _5kd_Memory_0.ram, _5ke.fdt, _5ke.fdx, _5ke.fnm, _5ke.nvd, _5ke.nvm, _5ke.si, _5ke_Memory_0.ram, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf_Memory_0.ram, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_2]}]> but was:<[{indexVersion=1423929401703,generation=3,filelist=[_5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5kh.fdt, _5kh.fdx, _5kh.fnm, _5kh.nvd, _5kh.nvm, _5kh.si, _5kh_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_3]}, {indexVersion=1423929401703,generation=2,filelist=[_5in.fdt, _5in.fdx, _5in.fnm, _5in.nvd, _5in.nvm, _5in.si, _5in_Memory_0.ram, _5jv.fdt, _5jv.fdx, _5jv.fnm, _5jv.nvd, _5jv.nvm, _5jv.si, _5jv_Memory_0.ram, _5jw.cfe, _5jw.cfs, _5jw.si, _5jx.fdt, _5jx.fdx, _5jx.fnm, _5jx.nvd, _5jx.nvm, _5jx.si, _5jx_Memory_0.ram, _5jy.fdt, _5jy.fdx, _5jy.fnm, _5jy.nvd, _5jy.nvm, _5jy.si, _5jy_Memory_0.ram, _5jz.fdt, _5jz.fdx, _5jz.fnm, _5jz.nvd, _5jz.nvm, _5jz.si, _5jz_Memory_0.ram, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0_Memory_0.ram, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1_Memory_0.ram, _5k2.fdt, _5k2.fdx, _5k2.fnm, _5k2.nvd, _5k2.nvm, _5k2.si, _5k2_Memory_0.ram, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3_Memory_0.ram, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4_Memory_0.ram, _5k5.fdt, _5k5.fdx, _5k5.fnm, _5k5.nvd, _5k5.nvm, _5k5.si, _5k5_Memory_0.ram, _5k6.fdt, _5k6.fdx, _5k6.fnm, _5k6.nvd, _5k6.nvm, _5k6.si, _5k6_Memory_0.ram, _5k7.fdt, _5k7.fdx, _5k7.fnm, _5k7.nvd, _5k7.nvm, _5k7.si, _5k7_Memory_0.ram, _5k8.fdt, _5k8.fdx, _5k8.fnm, _5k8.nvd, _5k8.nvm, _5k8.si, _5k8_Memory_0.ram, _5k9.fdt, _5k9.fdx, _5k9.fnm, _5k9.nvd, _5k9.nvm, _5k9.si, _5k9_Memory_0.ram, _5ka.fdt, _5ka.fdx, _5ka.fnm, _5ka.nvd, _5ka.nvm, _5ka.si, _5ka_Memory_0.ram, _5kb.fdt, _5kb.fdx, _5kb.fnm, _5kb.nvd, _5kb.nvm, _5kb.si, _5kb_Memory_0.ram, _5kc.fdt, _5kc.fdx, _5kc.fnm, _5kc.nvd, _5kc.nvm, _5kc.si, _5kc_Memory_0.ram, _5kd.fdt, _5kd.fdx, _5kd.fnm, _5kd.nvd, _5kd.nvm, _5kd.si, _5kd_Memory_0.ram, _5ke.fdt, _5ke.fdx, _5ke.fnm, _5ke.nvd, _5ke.nvm, _5ke.si, _5ke_Memory_0.ram, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf_Memory_0.ram, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1423929401703,generation=2,filelist=[_5in.fdt, _5in.fdx, _5in.fnm, _5in.nvd, _5in.nvm, _5in.si, _5in_Memory_0.ram, _5jv.fdt, _5jv.fdx, _5jv.fnm, _5jv.nvd, _5jv.nvm, _5jv.si, _5jv_Memory_0.ram, _5jw.cfe, _5jw.cfs, _5jw.si, _5jx.fdt, _5jx.fdx, _5jx.fnm, _5jx.nvd, _5jx.nvm, _5jx.si, _5jx_Memory_0.ram, _5jy.fdt, _5jy.fdx, _5jy.fnm, _5jy.nvd, _5jy.nvm, _5jy.si, _5jy_Memory_0.ram, _5jz.fdt, _5jz.fdx, _5jz.fnm, _5jz.nvd, _5jz.nvm, _5jz.si, _5jz_Memory_0.ram, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0_Memory_0.ram, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1_Memory_0.ram, _5k2.fdt, _5k2.fdx, _5k2.fnm, _5k2.nvd, _5k2.nvm, _5k2.si, _5k2_Memory_0.ram, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3_Memory_0.ram, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4_Memory_0.ram, _5k5.fdt, _5k5.fdx, _5k5.fnm, _5k5.nvd, _5k5.nvm, _5k5.si, _5k5_Memory_0.ram, _5k6.fdt, _5k6.fdx, _5k6.fnm, _5k6.nvd, _5k6.nvm, _5k6.si, _5k6_Memory_0.ram, _5k7.fdt, _5k7.fdx, _5k7.fnm, _5k7.nvd, _5k7.nvm, _5k7.si, _5k7_Memory_0.ram, _5k8.fdt, _5k8.fdx, _5k8.fnm, _5k8.nvd, _5k8.nvm, _5k8.si, _5k8_Memory_0.ram, _5k9.fdt, _5k9.fdx, _5k9.fnm, _5k9.nvd, _5k9.nvm, _5k9.si, _5k9_Memory_0.ram, _5ka.fdt, _5ka.fdx, _5ka.fnm, _5ka.nvd, _5ka.nvm, _5ka.si, _5ka_Memory_0.ram, _5kb.fdt, _5kb.fdx, _5kb.fnm, _5kb.nvd, _5kb.nvm, _5kb.si, _5kb_Memory_0.ram, _5kc.fdt, _5kc.fdx, _5kc.fnm, _5kc.nvd, _5kc.nvm, _5kc.si, _5kc_Memory_0.ram, _5kd.fdt, _5kd.fdx, _5kd.fnm, _5kd.nvd, _5kd.nvm, _5kd.si, _5kd_Memory_0.ram, _5ke.fdt, _5ke.fdx, _5ke.fnm, _5ke.nvd, _5ke.nvm, _5ke.si, _5ke_Memory_0.ram, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf_Memory_0.ram, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_2]}]> but was:<[{indexVersion=1423929401703,generation=3,filelist=[_5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5kh.fdt, _5kh.fdx, _5kh.fnm, _5kh.nvd, _5kh.nvm, _5kh.si, _5kh_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_3]}, {indexVersion=1423929401703,generation=2,filelist=[_5in.fdt, _5in.fdx, _5in.fnm, _5in.nvd, _5in.nvm, _5in.si, _5in_Memory_0.ram, _5jv.fdt, _5jv.fdx, _5jv.fnm, _5jv.nvd, _5jv.nvm, _5jv.si, _5jv_Memory_0.ram, _5jw.cfe, _5jw.cfs, _5jw.si, _5jx.fdt, _5jx.fdx, _5jx.fnm, _5jx.nvd, _5jx.nvm, _5jx.si, _5jx_Memory_0.ram, _5jy.fdt, _5jy.fdx, _5jy.fnm, _5jy.nvd, _5jy.nvm, _5jy.si, _5jy_Memory_0.ram, _5jz.fdt, _5jz.fdx, _5jz.fnm, _5jz.nvd, _5jz.nvm, _5jz.si, _5jz_Memory_0.ram, _5k0.fdt, _5k0.fdx, _5k0.fnm, _5k0.nvd, _5k0.nvm, _5k0.si, _5k0_Memory_0.ram, _5k1.fdt, _5k1.fdx, _5k1.fnm, _5k1.nvd, _5k1.nvm, _5k1.si, _5k1_Memory_0.ram, _5k2.fdt, _5k2.fdx, _5k2.fnm, _5k2.nvd, _5k2.nvm, _5k2.si, _5k2_Memory_0.ram, _5k3.fdt, _5k3.fdx, _5k3.fnm, _5k3.nvd, _5k3.nvm, _5k3.si, _5k3_Memory_0.ram, _5k4.fdt, _5k4.fdx, _5k4.fnm, _5k4.nvd, _5k4.nvm, _5k4.si, _5k4_Memory_0.ram, _5k5.fdt, _5k5.fdx, _5k5.fnm, _5k5.nvd, _5k5.nvm, _5k5.si, _5k5_Memory_0.ram, _5k6.fdt, _5k6.fdx, _5k6.fnm, _5k6.nvd, _5k6.nvm, _5k6.si, _5k6_Memory_0.ram, _5k7.fdt, _5k7.fdx, _5k7.fnm, _5k7.nvd, _5k7.nvm, _5k7.si, _5k7_Memory_0.ram, _5k8.fdt, _5k8.fdx, _5k8.fnm, _5k8.nvd, _5k8.nvm, _5k8.si, _5k8_Memory_0.ram, _5k9.fdt, _5k9.fdx, _5k9.fnm, _5k9.nvd, _5k9.nvm, _5k9.si, _5k9_Memory_0.ram, _5ka.fdt, _5ka.fdx, _5ka.fnm, _5ka.nvd, _5ka.nvm, _5ka.si, _5ka_Memory_0.ram, _5kb.fdt, _5kb.fdx, _5kb.fnm, _5kb.nvd, _5kb.nvm, _5kb.si, _5kb_Memory_0.ram, _5kc.fdt, _5kc.fdx, _5kc.fnm, _5kc.nvd, _5kc.nvm, _5kc.si, _5kc_Memory_0.ram, _5kd.fdt, _5kd.fdx, _5kd.fnm, _5kd.nvd, _5kd.nvm, _5kd.si, _5kd_Memory_0.ram, _5ke.fdt, _5ke.fdx, _5ke.fnm, _5ke.nvd, _5ke.nvm, _5ke.si, _5ke_Memory_0.ram, _5kf.fdt, _5kf.fdx, _5kf.fnm, _5kf.nvd, _5kf.nvm, _5kf.si, _5kf_Memory_0.ram, _5kg.fdt, _5kg.fdx, _5kg.fnm, _5kg.nvd, _5kg.nvm, _5kg.si, _5kg_Memory_0.ram, _5ki.fdt, _5ki.fdx, _5ki.fnm, _5ki.nvd, _5ki.nvm, _5ki.si, _5ki_Memory_0.ram, _5kj.fdt, _5kj.fdx, _5kj.fnm, _5kj.nvd, _5kj.nvm, _5kj.si, _5kj_Memory_0.ram, _5kk.fdt, _5kk.fdx, _5kk.fnm, _5kk.nvd, _5kk.nvm, _5kk.si, _5kk_Memory_0.ram, _5kl.fdt, _5kl.fdx, _5kl.fnm, _5kl.nvd, _5kl.nvm, _5kl.si, _5kl_Memory_0.ram, _5km.fdt, _5km.fdx, _5km.fnm, _5km.nvd, _5km.nvm, _5km.si, _5km_Memory_0.ram, _5kn.fdt, _5kn.fdx, _5kn.fnm, _5kn.nvd, _5kn.nvm, _5kn.si, _5kn_Memory_0.ram, _5ko.fdt, _5ko.fdx, _5ko.fnm, _5ko.nvd, _5ko.nvm, _5ko.si, _5ko_Memory_0.ram, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:59A6F9331977C8B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1142)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.spelling.SpellCheckCollatorTest.testEstimatedHitCounts

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:11F6CA96E4E06258]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:748)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:715)
	at org.apache.solr.spelling.SpellCheckCollatorTest.testEstimatedHitCounts(SpellCheckCollatorTest.java:530)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//lst[@name='spellcheck']/lst[@name='collations']/lst[@name='collation']/int[@name='hits' and 6 <= . and . <= 10]
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">7</int></lst><result name="response" numFound="0" start="0"></result><lst name="spellcheck"><lst name="suggestions"><lst name="everother"><int name="numFound">1</int><int name="startOffset">9</int><int name="endOffset">18</int><arr name="suggestion"><str>everyother</str></arr></lst></lst><lst name="collations"><lst name="collation"><str name="collationQuery">teststop:everyother</str><int name="hits">12</int><lst name="misspellingsAndCorrections"><str name="everother">everyother</str></lst></lst></lst></lst>
</response>

	request was:spellcheck=true&spellcheck.dictionary=direct&spellcheck.count=1&spellcheck.collate=true&spellcheck.maxCollationTries=1&spellcheck.maxCollations=1&spellcheck.collateExtendedResults=true&qt=spellCheckCompRH&q=teststop%3Aeverother&spellcheck.collateMaxCollectDocs=5
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:741)
	... 42 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32689/iauhr/fm/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32689/iauhr/fm/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10199 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/init-core-data-001
   [junit4]   2> 497192 T1099 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /hr_/
   [junit4]   2> 497205 T1099 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 497206 T1100 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 497307 T1099 oasc.ZkTestServer.run start zk server on port:62107
   [junit4]   2> 497308 T1099 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 497310 T1099 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497329 T1107 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6615ae0e name:ZooKeeperConnection Watcher:127.0.0.1:62107 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497330 T1099 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497332 T1099 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 497332 T1099 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 497338 T1099 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 497339 T1099 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497342 T1110 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3078cee0 name:ZooKeeperConnection Watcher:127.0.0.1:62107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497342 T1099 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497343 T1099 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 497344 T1099 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 497349 T1099 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 497356 T1099 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 497362 T1099 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 497370 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 497372 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 497380 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 497380 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 497386 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 497389 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 497400 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 497404 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 497413 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 497414 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 497417 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 497417 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 497421 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 497423 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 497431 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 497433 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 497443 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 497444 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 497450 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 497451 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 497454 T1099 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 497455 T1099 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 499999 T1099 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores/collection1
   [junit4]   2> 500004 T1099 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 500011 T1099 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62114
   [junit4]   2> 500012 T1099 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 500012 T1099 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 500012 T1099 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001
   [junit4]   2> 500013 T1099 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/'
   [junit4]   2> 500071 T1099 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/solr.xml
   [junit4]   2> 500146 T1099 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores
   [junit4]   2> 500146 T1099 oasc.CoreContainer.<init> New CoreContainer 762536149
   [junit4]   2> 500147 T1099 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/]
   [junit4]   2> 500148 T1099 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 500148 T1099 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 500149 T1099 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 500149 T1099 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 500149 T1099 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 500149 T1099 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 500150 T1099 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 500150 T1099 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 500150 T1099 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 500151 T1099 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 500151 T1099 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 500152 T1099 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 500152 T1099 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 500152 T1099 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 500153 T1099 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 500153 T1099 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62107/solr
   [junit4]   2> 500153 T1099 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 500154 T1099 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 500155 T1099 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500169 T1122 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@229a5402 name:ZooKeeperConnection Watcher:127.0.0.1:62107 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500170 T1099 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500170 T1099 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 500173 T1099 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500174 T1125 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4509d321 name:ZooKeeperConnection Watcher:127.0.0.1:62107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500175 T1099 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500176 T1099 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 500181 T1099 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 500184 T1099 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 500187 T1099 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 500189 T1099 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 500192 T1099 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 500193 T1099 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62113_hr_
   [junit4]   2> 500194 T1099 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62113_hr_
   [junit4]   2> 500197 T1099 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 500198 T1099 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 500200 T1099 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 500202 T1099 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62113_hr_
   [junit4]   2> 500203 T1099 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 500205 T1099 oasc.Overseer.start Overseer (id=93318577221795843-127.0.0.1:62113_hr_-n_0000000000) starting
   [junit4]   2> 500207 T1099 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 500215 T1099 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 500216 T1127 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 500217 T1099 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 500219 T1099 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 500220 T1099 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 500224 T1126 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 500225 T1099 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores
   [junit4]   2> 500228 T1099 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores/collection1/
   [junit4]   2> 500229 T1099 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 500230 T1129 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 500230 T1129 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 500231 T1129 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 500231 T1125 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 500234 T1126 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62113/hr_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:62113_hr_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 500235 T1126 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62113/hr_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:62113_hr_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 500235 T1126 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 500235 T1126 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 500237 T1125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 501231 T1129 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 501232 T1129 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 501233 T1129 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 501233 T1129 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 501235 T1129 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 501235 T1129 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores/collection1/'
   [junit4]   2> 501267 T1129 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 501273 T1129 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 501296 T1129 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 501343 T1129 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 501446 T1129 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 501449 T1129 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 501469 T1129 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 502072 T1129 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 502101 T1129 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 502106 T1129 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 502127 T1129 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502137 T1129 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502147 T1129 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502149 T1129 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502150 T1129 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502151 T1129 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502152 T1129 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502152 T1129 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502153 T1129 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 502153 T1129 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 502154 T1129 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/tempDir-001/control/data/
   [junit4]   2> 502154 T1129 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@319edff2
   [junit4]   2> 502155 T1129 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/tempDir-001/control/data
   [junit4]   2> 502155 T1129 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/tempDir-001/control/data/index/
   [junit4]   2> 502155 T1129 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 502156 T1129 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 204D74A341DF7288-001/tempDir-001/control/data/index
   [junit4]   2> 502157 T1129 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=425167565, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 502157 T1129 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1776388d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@527abd03),segFN=segments_1,generation=1}
   [junit4]   2> 502158 T1129 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502165 T1129 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 502166 T1129 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 502166 T1129 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 502166 T1129 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502166 T1129 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502167 T1129 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 502167 T1129 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502167 T1129 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502168 T1129 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 502169 T1129 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 502169 T1129 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 502170 T1129 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 502171 T1129 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 502172 T1129 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 502172 T1129 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 502173 T1129 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 502174 T1129 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 502175 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 502177 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 502178 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 502179 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 502180 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 502180 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 502181 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 502182 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 502183 T1129 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 502183 T1129 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 502278 T1129 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502284 T1129 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502289 T1129 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502295 T1129 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502299 T1129 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 502315 T1129 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 502315 T1129 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 502316 T1129 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=97.38671875, floorSegmentMB=1.333984375, forceMergeDeletesPctAllowed=12.15872251998703, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 502317 T1129 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1776388d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@527abd03),segFN=segments_1,generation=1}
   [junit4]   2> 502318 T1129 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502318 T1129 oass.SolrIndexSearcher.<init> Opening Searcher@73ba4d49[collection1] main
   [junit4]   2> 502319 T1129 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 502320 T1129 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 502321 T1129 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 502322 T1129 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 502322 T1129 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 502322 T1129 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502323 T1129 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 502323 T1129 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502324 T1129 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 502324 T1129 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 502325 T1130 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73ba4d49[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 502326 T1129 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 502327 T1133 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62113/hr_ collection:control_collection shard:shard1
   [junit4]   2> 502328 T1099 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 502328 T1099 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 502328 T1133 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 502336 T1133 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 502337 T1099 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 502338 T1099 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 502340 T1137 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6701838c name:ZooKeeperConnection Watcher:127.0.0.1:62107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 502340 T1125 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 502341 T1133 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 502341 T1099 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 502341 T1133 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C213 name=collection1 org.apache.solr.core.SolrCore@3ae5b825 url=http://127.0.0.1:62113/hr_/collection1 node=127.0.0.1:62113_hr_ C213_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:62113/hr_, core=collection1, state=down, node_name=127.0.0.1:62113_hr_}
   [junit4]   2> 502342 T1133 C213 P62113 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62113/hr_/collection1/
   [junit4]   2> 502342 T1126 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 502342 T1099 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 502343 T1133 C213 P62113 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 502344 T1133 C213 P62113 oasc.SyncStrategy.syncToMe http://127.0.0.1:62113/hr_/collection1/ has no replicas
   [junit4]   2> 502344 T1133 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62113/hr_/collection1/ shard1
   [junit4]   2> 502345 T1133 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 502345 T1099 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 502346 T1125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncCo

[...truncated too long message...]

/127.0.0.1:61806/dqvtr/uo
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test -Dtests.seed=204D74A341DF7288 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ga_IE -Dtests.timezone=Etc/UTC -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   93.6s J3 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([204D74A341DF7288:A8194B79EF231F70]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2221265 T9650 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderFailoverAfterPartitionTest 204D74A341DF7288-001
   [junit4]   2> 93557 T9649 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom), _version_=Lucene50(blocksize=128), range_facet_l=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=ga_IE, timezone=Etc/UTC
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=10343848,total=412090368
   [junit4]   2> NOTE: All tests run in this JVM: [ZkCLITest, QueryResultKeyTest, UnloadDistributedZkTest, OutOfBoxZkACLAndCredentialsProvidersTest, CustomCollectionTest, MultiTermTest, DistributedMLTComponentTest, TestNonDefinedSimilarityFactory, CloudMLTQParserTest, HdfsBasicDistributedZk2Test, SuggesterWFSTTest, SolrTestCaseJ4Test, ClusterStateTest, HdfsBasicDistributedZkTest, FileUtilsTest, DocValuesTest, TestManagedSchema, ZkStateWriterTest, TestJmxMonitoredMap, TestStressUserVersions, TestBulkSchemaAPI, TestGroupingSearch, TestPostingsSolrHighlighter, TestIndexingPerformance, SimpleCollectionCreateDeleteTest, TestCloudManagedSchemaConcurrent, TestSchemaManager, DefaultValueUpdateProcessorTest, HttpPartitionTest, TestUtils, TestFiltering, TestConfigSets, TestPivotHelperCode, TestWordDelimiterFilterFactory, ShardRoutingTest, SpatialHeatmapFacetsTest, TestNoOpRegenerator, BinaryUpdateRequestHandlerTest, ClusterStateUpdateTest, TestWriterPerf, ConnectionManagerTest, TestJmxIntegration, TestSchemaSimilarityResource, TermsComponentTest, DistribJoinFromCollectionTest, HdfsRecoveryZkTest, TestLMDirichletSimilarityFactory, TestCodecSupport, AddSchemaFieldsUpdateProcessorFactoryTest, TestIndexSearcher, CoreAdminHandlerTest, TestShortCircuitedRequests, CurrencyFieldXmlFileTest, AnalysisAfterCoreReloadTest, TestFieldTypeCollectionResource, ActionThrottleTest, DateMathParserTest, BasicDistributedZk2Test, SyncSliceTest, OverseerTest, BasicZkTest, TestDistributedGrouping, TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, ZkControllerTest, TestStressReorder, TestReload, StatsComponentTest, TestFunctionQuery, TestLazyCores, OverseerCollectionProcessorTest, ShowFileRequestHandlerTest, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, PolyFieldTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestRemoteStreaming, SolrInfoMBeanTest, DisMaxRequestHandlerTest, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, JsonLoaderTest, TestQuerySenderListener, TestQuerySenderNoQuery, SolrIndexConfigTest, TestDocSet, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, TestLuceneMatchVersion, TestCharFilters, TestDFRSimilarityFactory, TestDefaultSimilarityFactory, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, DeleteReplicaTest, DistributedQueueTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J3 in 93.85s, 1 test, 1 error <<< FAILURES!

[...truncated 306 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1864 tests, 6 errors, 1 failure, 56 ignored (9 assumptions)

Total time: 158 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 715446534 bytes
Compression is 0.0%
Took 2 min 14 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 762 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/762/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:24829/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24829/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9942 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/init-core-data-001
   [junit4]   2> 259660 T1512 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 259667 T1512 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 259668 T1513 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 259768 T1512 oasc.ZkTestServer.run start zk server on port:24788
   [junit4]   2> 259769 T1512 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 259770 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 259775 T1520 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6996bb70 name:ZooKeeperConnection Watcher:127.0.0.1:24788 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 259776 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 259776 T1512 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 259777 T1512 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 259780 T1512 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 259781 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 259782 T1523 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4bb13b2e name:ZooKeeperConnection Watcher:127.0.0.1:24788/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 259783 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 259783 T1512 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 259784 T1512 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 259786 T1512 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 259787 T1512 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 259789 T1512 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 259791 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 259791 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 259794 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 259795 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 259797 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 259798 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 259800 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 259801 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 259803 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 259804 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 259806 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 259806 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 259808 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 259809 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 259811 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 259812 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 259814 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 259815 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 259817 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 259817 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 259820 T1512 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 259820 T1512 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 261253 T1512 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores/collection1
   [junit4]   2> 261258 T1512 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 261261 T1512 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24797
   [junit4]   2> 261262 T1512 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 261262 T1512 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 261262 T1512 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001
   [junit4]   2> 261263 T1512 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/'
   [junit4]   2> 261309 T1512 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/solr.xml
   [junit4]   2> 261364 T1512 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores
   [junit4]   2> 261364 T1512 oasc.CoreContainer.<init> New CoreContainer 1750628002
   [junit4]   2> 261365 T1512 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/]
   [junit4]   2> 261366 T1512 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 261366 T1512 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 261366 T1512 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 261366 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 261367 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 261367 T1512 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 261367 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 261367 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 261368 T1512 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 261368 T1512 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 261368 T1512 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 261369 T1512 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 261369 T1512 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 261369 T1512 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 261370 T1512 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 261370 T1512 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:24788/solr
   [junit4]   2> 261370 T1512 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 261371 T1512 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 261371 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261374 T1535 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56e6e933 name:ZooKeeperConnection Watcher:127.0.0.1:24788 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261374 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261375 T1512 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 261376 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261377 T1538 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71dd9f6c name:ZooKeeperConnection Watcher:127.0.0.1:24788/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261377 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261379 T1512 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 261380 T1512 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 261382 T1512 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 261384 T1512 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 261385 T1512 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 261387 T1512 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 261388 T1512 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24796_
   [junit4]   2> 261389 T1512 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24796_
   [junit4]   2> 261390 T1512 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 261392 T1512 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 261393 T1512 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 261394 T1512 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:24796_
   [junit4]   2> 261394 T1512 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 261396 T1512 oasc.Overseer.start Overseer (id=93312857059557379-127.0.0.1:24796_-n_0000000000) starting
   [junit4]   2> 261397 T1512 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 261403 T1512 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 261403 T1540 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 261403 T1512 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 261405 T1512 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 261406 T1512 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 261408 T1539 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 261409 T1512 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores
   [junit4]   2> 261411 T1512 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores/collection1/
   [junit4]   2> 261411 T1512 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 261412 T1542 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 261412 T1542 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 261413 T1542 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 261413 T1538 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 261415 T1539 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:24796_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24796",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 261416 T1539 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:24796_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24796",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 261416 T1539 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 261416 T1539 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 261417 T1538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 262414 T1542 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 262415 T1542 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 262415 T1542 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 262416 T1542 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 262416 T1542 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 262417 T1542 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores/collection1/'
   [junit4]   2> 262429 T1542 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 262431 T1542 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 262439 T1542 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 262454 T1542 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 262486 T1542 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 262487 T1542 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 262496 T1542 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 262832 T1542 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 262844 T1542 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 262847 T1542 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 262866 T1542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 262870 T1542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 262875 T1542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 262876 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 262876 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 262877 T1542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 262878 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 262878 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 262878 T1542 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 262879 T1542 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 262879 T1542 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/control/data/
   [junit4]   2> 262879 T1542 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f8c4a08
   [junit4]   2> 262880 T1542 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/control/data
   [junit4]   2> 262880 T1542 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/control/data/index/
   [junit4]   2> 262881 T1542 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 262881 T1542 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/control/data/index
   [junit4]   2> 262882 T1542 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2d159375
   [junit4]   2> 262883 T1542 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6383fd72 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e747a68),segFN=segments_1,generation=1}
   [junit4]   2> 262883 T1542 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 262889 T1542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 262890 T1542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 262890 T1542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 262890 T1542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 262891 T1542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 262891 T1542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 262892 T1542 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 262892 T1542 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 262892 T1542 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 262893 T1542 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 262894 T1542 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 262894 T1542 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 262895 T1542 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 262895 T1542 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 262896 T1542 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 262897 T1542 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 262897 T1542 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 262898 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 262899 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 262900 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 262900 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 262901 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 262901 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 262902 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 262903 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 262903 T1542 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 262904 T1542 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 262923 T1542 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 262925 T1542 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 262927 T1542 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 262929 T1542 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 262936 T1542 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 262939 T1542 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 262939 T1542 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 262940 T1542 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=49.05078125, floorSegmentMB=0.662109375, forceMergeDeletesPctAllowed=25.489214020250575, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13946092033574342
   [junit4]   2> 262941 T1542 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6383fd72 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e747a68),segFN=segments_1,generation=1}
   [junit4]   2> 262941 T1542 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 262941 T1542 oass.SolrIndexSearcher.<init> Opening Searcher@dbe5c55[collection1] main
   [junit4]   2> 262941 T1542 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 262942 T1542 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 262943 T1542 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 262943 T1542 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 262943 T1542 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 262943 T1542 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 262944 T1542 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 262944 T1542 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 262944 T1542 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 262945 T1542 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 262945 T1542 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 262946 T1543 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@dbe5c55[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 262946 T1546 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24796 collection:control_collection shard:shard1
   [junit4]   2> 262946 T1512 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 262947 T1512 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 262947 T1546 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 262950 T1512 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 262951 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 262952 T1546 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 262952 T1550 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7537b1ec name:ZooKeeperConnection Watcher:127.0.0.1:24788/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 262953 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 262953 T1512 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 262953 T1538 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 262954 T1546 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 262954 T1546 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 262954 T1512 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 262954 T1539 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C38 name=collection1 org.apache.solr.core.SolrCore@32fbc784 url=http://127.0.0.1:24796/collection1 node=127.0.0.1:24796_ C38_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:24796_, base_url=http://127.0.0.1:24796, core=collection1, state=down}
   [junit4]   2> 262954 T1546 C38 P24796 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:24796/collection1/
   [junit4]   2> 262955 T1546 C38 P24796 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 262955 T1546 C38 P24796 oasc.SyncStrategy.syncToMe http://127.0.0.1:24796/collection1/ has no replicas
   [junit4]   2> 262955 T1546 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:24796/collection1/ shard1
   [junit4]   2> 262956 T1546 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 262956 T1550 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 262956 T1538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 262957 T1512 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 262959 T1551 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 262960 T1539 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24796",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 263063 T1550 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 263112 T1546 oasc.ZkController.register We are http://127.0.0.1:24796/collection1/ and leader is http://127.0.0.1:24796/collection1/
   [junit4]   2> 263113 T1546 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24796
   [junit4]   2> 263113 T1546 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 263113 T1546 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 263113 T1546 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 263115 T1538 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 263116 T1539 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:24796_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24796",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 263117 T1539 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:24796_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24796",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 263220 T1538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 263220 T1550 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264811 T1512 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores/collection1
   [junit4]   2> 264815 T1512 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 264817 T1512 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 264820 T1512 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24809
   [junit4]   2> 264820 T1512 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 264821 T1512 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 264821 T1512 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001
   [junit4]   2> 264822 T1512 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/'
   [junit4]   2> 264878 T1512 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/solr.xml
   [junit4]   2> 264953 T1512 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores
   [junit4]   2> 264954 T1512 oasc.CoreContainer.<init> New CoreContainer 1744749159
   [junit4]   2> 264954 T1512 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/]
   [junit4]   2> 264956 T1512 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 264956 T1512 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 264956 T1512 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 264957 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 264957 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 264957 T1512 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 264958 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 264958 T1512 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 264959 T1512 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 264959 T1512 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 264959 T1512 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 264960 T1512 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 264960 T1512 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 264961 T1512 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 264961 T1512 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 264962 T1512 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:24788/solr
   [junit4]   2> 264962 T1512 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 264962 T1512 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 264963 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 264965 T1563 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@166c7757 name:ZooKeeperConnection Watcher:127.0.0.1:24788 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 264966 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 264967 T1512 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 264969 T1512 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 264970 T1566 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b533d92 name:ZooKeeperConnection Watcher:127.0.0.1:24788/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 264970 T1512 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 264975 T1512 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 265979 T1512 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24808_
   [junit4]   2> 265980 T1512 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24808_
   [junit4]   2> 265983 T1512 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 265986 T1512 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores
   [junit4]   2> 265988 T1512 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores/collection1/
   [junit4]   2> 265989 T1512 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 265990 T1567 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 265991 T1567 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 265992 T1567 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 265992 T1538 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 265993 T1539 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:24808_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24808",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 265993 T1539 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:24808_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24808",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 265993 T1539 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 265994 T1539 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 265995 T1538 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 265995 T1550 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 265995 T1566 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 266992 T1567 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 266993 T1567 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 266993 T1567 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 266994 T1567 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 266994 T1567 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 266994 T1567 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores/collection1/'
   [junit4]   2> 267007 T1567 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 267009 T1567 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 267018 T1567 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 267033 T1567 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 267067 T1567 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 267068 T1567 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 267077 T1567 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 267347 T1567 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 267355 T1567 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 267357 T1567 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 267370 T1567 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 267374 T1567 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 267377 T1567 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 267377 T1567 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 267378 T1567 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 267378 T1567 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 267379 T1567 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 267379 T1567 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 267379 T1567 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 267379 T1567 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 267380 T1567 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/jetty1/
   [junit4]   2> 267380 T1567 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f8c4a08
   [junit4]   2> 267381 T1567 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/jetty1
   [junit4]   2> 267381 T1567 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/jetty1/index/
   [junit4]   2> 267381 T1567 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 267381 T1567 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FB6D81706C81D3F-001/tempDir-001/jetty1/index
   [junit4]   2> 267382 T1567 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@20368d40
   [junit4]   2> 267383 T1567 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63bdeae lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@449874b4),segFN=segments_1,generation=1}
   [junit4]   2> 267383 T1567 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 267388 T1567 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 267388 T1567 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 267389 T1567 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 267389 T1567 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 267389 T1567 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 267389 T1567 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 267390 T1567 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 267390 T1567 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 267390 T1567 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 267391 T1567 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 267391 T1567 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 267392 T1567 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 267392 T1567 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 267393 T1567 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 267393 T1567 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 267394 T1567 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 267395 T1567 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 267395 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 267396 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 267397 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 267397 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 267398 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 267398 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 267399 T1567 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 267399 T15

[...truncated too long message...]

ome/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_MA -Dtests.timezone=Etc/GMT-3 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   74.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FB6D81706C81D3F:87E2E7CDA83470C7]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2524205 T10048 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest FB6D81706C81D3F-001
   [junit4]   2> 74742 T10047 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=PostingsFormat(name=Memory doPackFST= true), multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ar_MA, timezone=Etc/GMT-3
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=285098632,total=435159040
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedQueryComponentCustomSortTest, UpdateParamsTest, TestStressLucene, AnalyticsMergeStrategyTest, TestCollapseQParserPlugin, CopyFieldTest, SchemaVersionSpecificBehaviorTest, DistributedIntervalFacetingTest, TestCloudPivotFacet, AssignTest, DistribCursorPagingTest, DisMaxRequestHandlerTest, SaslZkACLProviderTest, CachingDirectoryFactoryTest, AtomicUpdatesTest, ZkControllerTest, TestReplicationHandlerBackup, DistributedTermsComponentTest, IndexSchemaTest, TestSearcherReuse, TestManagedSchemaDynamicFieldResource, EchoParamsTest, TestLMJelinekMercerSimilarityFactory, TestTolerantSearch, VMParamsZkACLAndCredentialsProvidersTest, TestCollectionAPI, TestInitQParser, TestBinaryResponseWriter, TestAnalyzeInfixSuggestions, PingRequestHandlerTest, SSLMigrationTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, CursorMarkTest, AutoCommitTest, ExitableDirectoryReaderTest, TestSolrDeletionPolicy1, TestStressRecovery, RankQueryTest, CoreMergeIndexesAdminHandlerTest, TestFieldCollectionResource, TestPhraseSuggestions, FieldMutatingUpdateProcessorTest, HdfsChaosMonkeySafeLeaderTest, DirectSolrConnectionTest, SolrInfoMBeanTest, TestSolr4Spatial, TestMiniSolrCloudCluster, FullSolrCloudDistribCmdsTest, TestRebalanceLeaders, TestStressVersions, TestLFUCache, MigrateRouteKeyTest, DocumentAnalysisRequestHandlerTest, TestManagedSchemaFieldResource, TestSolr4Spatial2, QueryElevationComponentTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, ClusterStateUpdateTest, TestRandomFaceting, ZkSolrClientTest, TestZkChroot, TestFaceting, TestRecovery, TestStressReorder, TestRangeQuery, TestCoreContainer, SolrCoreTest, TestGroupingSearch, TestBadConfig, ShowFileRequestHandlerTest, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, SuggesterTSTTest, TestTrie, SpatialFilterTest, WordBreakSolrSpellCheckerTest, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, StandardRequestHandlerTest, DirectSolrSpellCheckerTest, TestOmitPositions, XmlUpdateRequestHandlerTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, JSONWriterTest, MBeansHandlerTest, TestAnalyzedSuggestions, JsonLoaderTest, TestPHPSerializedResponseWriter, TestComponentsName, TestQuerySenderListener, TestQuerySenderNoQuery, TestSolrDeletionPolicy2, TestBinaryField, MinimalSchemaTest, OutputWriterTest, TestFuzzyAnalyzedSuggestions, SpellPossibilityIteratorTest, TestCodecSupport, TestSweetSpotSimilarityFactory, TestDefaultSimilarityFactory, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, PreAnalyzedFieldTest, RAMDirectoryFactoryTest, TestDocumentBuilder, CircularListTest, TestRTGBase, CursorPagingTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, LeaderFailoverAfterPartitionTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 75.52s, 1 test, 1 error <<< FAILURES!

[...truncated 213 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1864 tests, 5 errors, 57 ignored (10 assumptions)

Total time: 149 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 682985531 bytes
Compression is 0.0%
Took 2 min 4 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 761 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/761/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33382/hn/n/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33382/hn/n/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33382/hn/n/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33251/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33251/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33251/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:12374/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12374/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59147/e_/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59147/e_/collection1
	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59147/e_/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17450/jb_sfp/vb/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17450/jb_sfp/vb/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17450/jb_sfp/vb/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 10102 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/init-core-data-001
   [junit4]   2> 246183 T1391 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /hn/n
   [junit4]   2> 246187 T1391 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 246188 T1392 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 246288 T1391 oasc.ZkTestServer.run start zk server on port:33367
   [junit4]   2> 246289 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 246290 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 246293 T1399 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@662416f5 name:ZooKeeperConnection Watcher:127.0.0.1:33367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 246293 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 246294 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 246294 T1391 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 246297 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 246298 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 246299 T1402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7db33444 name:ZooKeeperConnection Watcher:127.0.0.1:33367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 246299 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 246300 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 246300 T1391 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 246302 T1391 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 246303 T1391 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 246305 T1391 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 246306 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 246307 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 246310 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 246310 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 246312 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 246312 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 246314 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 246314 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 246316 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 246316 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 246318 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 246319 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 246320 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 246321 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 246322 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 246323 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 246324 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 246325 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 246326 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 246327 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 246328 T1391 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 246329 T1391 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 247976 T1391 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores/collection1
   [junit4]   2> 247981 T1391 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 247985 T1391 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33383
   [junit4]   2> 247986 T1391 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 247986 T1391 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 247986 T1391 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001
   [junit4]   2> 247987 T1391 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/'
   [junit4]   2> 248039 T1391 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/solr.xml
   [junit4]   2> 248099 T1391 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores
   [junit4]   2> 248100 T1391 oasc.CoreContainer.<init> New CoreContainer 75271612
   [junit4]   2> 248100 T1391 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/]
   [junit4]   2> 248101 T1391 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 248102 T1391 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 248102 T1391 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 248102 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 248103 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 248103 T1391 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 248104 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 248104 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 248104 T1391 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 248104 T1391 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 248105 T1391 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 248105 T1391 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 248106 T1391 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 248106 T1391 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 248106 T1391 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 248107 T1391 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33367/solr
   [junit4]   2> 248107 T1391 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 248107 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 248108 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 248110 T1414 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ff7b347 name:ZooKeeperConnection Watcher:127.0.0.1:33367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 248110 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 248111 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 248114 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 248115 T1417 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@172f7c0e name:ZooKeeperConnection Watcher:127.0.0.1:33367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 248116 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 248117 T1391 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 248120 T1391 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 248123 T1391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 248125 T1391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 248128 T1391 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 248131 T1391 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 248132 T1391 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33382_hn%2Fn
   [junit4]   2> 248133 T1391 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33382_hn%2Fn
   [junit4]   2> 248135 T1391 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 248137 T1391 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 248139 T1391 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 248141 T1391 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33382_hn%2Fn
   [junit4]   2> 248141 T1391 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 248143 T1391 oasc.Overseer.start Overseer (id=93307185052254211-127.0.0.1:33382_hn%2Fn-n_0000000000) starting
   [junit4]   2> 248145 T1391 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 248152 T1391 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 248153 T1419 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 248154 T1391 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 248156 T1391 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 248158 T1391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 248161 T1418 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 248162 T1391 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores
   [junit4]   2> 248164 T1391 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores/collection1/
   [junit4]   2> 248164 T1391 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 248165 T1421 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 248165 T1421 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 248166 T1417 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 248166 T1421 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 248168 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33382/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:33382_hn%2Fn",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 248169 T1418 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33382/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:33382_hn%2Fn",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 248169 T1418 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 248169 T1418 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 248171 T1417 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249166 T1421 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 249167 T1421 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 249167 T1421 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 249167 T1421 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 249168 T1421 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 249168 T1421 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores/collection1/'
   [junit4]   2> 249180 T1421 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 249182 T1421 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 249190 T1421 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 249204 T1421 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 249235 T1421 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 249236 T1421 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 249243 T1421 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 249496 T1421 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 249503 T1421 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 249505 T1421 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 249516 T1421 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 249519 T1421 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 249522 T1421 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 249523 T1421 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 249523 T1421 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 249523 T1421 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 249524 T1421 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 249524 T1421 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 249524 T1421 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 249525 T1421 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 249525 T1421 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/control/data/
   [junit4]   2> 249525 T1421 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ab75892
   [junit4]   2> 249526 T1421 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/control/data
   [junit4]   2> 249526 T1421 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/control/data/index/
   [junit4]   2> 249526 T1421 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 249526 T1421 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/control/data/index
   [junit4]   2> 249527 T1421 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3c5b517a
   [junit4]   2> 249527 T1421 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@f423f8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4659c680),segFN=segments_1,generation=1}
   [junit4]   2> 249527 T1421 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 249531 T1421 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 249531 T1421 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 249531 T1421 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 249532 T1421 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 249532 T1421 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 249532 T1421 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 249532 T1421 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 249533 T1421 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 249533 T1421 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 249533 T1421 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 249534 T1421 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 249534 T1421 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 249534 T1421 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 249535 T1421 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 249535 T1421 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 249535 T1421 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 249536 T1421 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 249536 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 249537 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 249537 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 249538 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 249538 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 249538 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 249539 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 249539 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 249539 T1421 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 249540 T1421 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 249552 T1421 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249553 T1421 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249554 T1421 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249555 T1421 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249557 T1421 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 249559 T1421 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 249559 T1421 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 249561 T1421 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=251102189, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 249561 T1421 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@f423f8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4659c680),segFN=segments_1,generation=1}
   [junit4]   2> 249561 T1421 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 249562 T1421 oass.SolrIndexSearcher.<init> Opening Searcher@7b067fd7[collection1] main
   [junit4]   2> 249562 T1421 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 249563 T1421 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 249563 T1421 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 249563 T1421 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 249564 T1421 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 249564 T1421 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 249564 T1421 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 249564 T1421 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 249565 T1421 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 249565 T1421 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 249565 T1422 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b067fd7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 249566 T1421 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 249567 T1425 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33382/hn/n collection:control_collection shard:shard1
   [junit4]   2> 249567 T1391 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 249567 T1391 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 249567 T1425 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 249570 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 249570 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 249572 T1429 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cf78c3d name:ZooKeeperConnection Watcher:127.0.0.1:33367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 249572 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 249572 T1425 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 249572 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 249573 T1391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 249574 T1417 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249574 T1425 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 249574 T1425 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C103 name=collection1 org.apache.solr.core.SolrCore@2c611f62 url=http://127.0.0.1:33382/hn/n/collection1 node=127.0.0.1:33382_hn%2Fn C103_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:33382/hn/n, state=down, node_name=127.0.0.1:33382_hn%2Fn}
   [junit4]   2> 249574 T1425 C103 P33382 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33382/hn/n/collection1/
   [junit4]   2> 249574 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 249574 T1425 C103 P33382 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 249575 T1425 C103 P33382 oasc.SyncStrategy.syncToMe http://127.0.0.1:33382/hn/n/collection1/ has no replicas
   [junit4]   2> 249575 T1425 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33382/hn/n/collection1/ shard1
   [junit4]   2> 249575 T1425 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 249576 T1391 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 249576 T1391 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 249576 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 249577 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 249578 T1432 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75e98791 name:ZooKeeperConnection Watcher:127.0.0.1:33367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 249578 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 249579 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 249579 T1417 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249580 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33382/hn/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 249584 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 249584 T1418 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 249585 T1417 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249585 T1429 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249589 T1433 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249590 T1418 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 249591 T1429 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249630 T1425 oasc.ZkController.register We are http://127.0.0.1:33382/hn/n/collection1/ and leader is http://127.0.0.1:33382/hn/n/collection1/
   [junit4]   2> 249631 T1425 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33382/hn/n
   [junit4]   2> 249631 T1425 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 249631 T1425 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 249631 T1425 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 249632 T1417 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249634 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33382/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:33382_hn%2Fn",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 249635 T1418 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33382/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:33382_hn%2Fn",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 249739 T1417 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249739 T1429 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 250716 T1391 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores/collection1
   [junit4]   2> 250720 T1391 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 250721 T1391 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 250723 T1391 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33405
   [junit4]   2> 250723 T1391 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 250724 T1391 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 250724 T1391 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001
   [junit4]   2> 250724 T1391 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/'
   [junit4]   2> 250771 T1391 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/solr.xml
   [junit4]   2> 250826 T1391 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores
   [junit4]   2> 250826 T1391 oasc.CoreContainer.<init> New CoreContainer 203632939
   [junit4]   2> 250827 T1391 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/]
   [junit4]   2> 250828 T1391 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 250828 T1391 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 250829 T1391 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 250829 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 250829 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 250830 T1391 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 250830 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 250830 T1391 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 250831 T1391 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 250831 T1391 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 250831 T1391 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 250832 T1391 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 250832 T1391 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 250833 T1391 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 250833 T1391 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 250834 T1391 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33367/solr
   [junit4]   2> 250834 T1391 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 250834 T1391 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 250835 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 250837 T1445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34575471 name:ZooKeeperConnection Watcher:127.0.0.1:33367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 250837 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 250838 T1391 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 250841 T1391 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 250843 T1448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14935302 name:ZooKeeperConnection Watcher:127.0.0.1:33367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 250843 T1391 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 250849 T1391 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 251854 T1391 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33404_hn%2Fn
   [junit4]   2> 251855 T1391 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33404_hn%2Fn
   [junit4]   2> 251859 T1391 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 251862 T1391 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores
   [junit4]   2> 251864 T1391 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores/collection1/
   [junit4]   2> 251864 T1391 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 251867 T1449 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 251867 T1449 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 251869 T1417 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 251870 T1449 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 251870 T1449 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 251870 T1418 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33404/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:33404_hn%2Fn",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 251871 T1449 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 251871 T1418 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33404/hn/n",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:33404_hn%2Fn",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 251871 T1418 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 251871 T1418 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 251872 T1449 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 251872 T1449 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 251873 T1418 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 251874 T1448 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 251874 T1448 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 252873 T1449 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 252874 T1449 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 252875 T1449 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 252875 T1449 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 252876 T1449 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 252876 T1449 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores/collection1/'
   [junit4]   2> 252892 T1449 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 252895 T1449 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 252906 T1449 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 252939 T1449 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 252985 T1449 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 252986 T1449 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 252997 T1449 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 253365 T1449 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 253376 T1449 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 253378 T1449 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 253392 T1449 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 253397 T1449 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 253401 T1449 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 253402 T1449 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 253402 T1449 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 253403 T1449 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 253403 T1449 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 253404 T1449 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 253404 T1449 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 253404 T1449 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 253405 T1449 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/jetty1/
   [junit4]   2> 253405 T1449 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ab75892
   [junit4]   2> 253406 T1449 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/jetty1
   [junit4]   2> 253406 T1449 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/jetty1/index/
   [junit4]   2> 253406 T1449 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 253407 T1449 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 95E8F46B89D049DF-001/tempDir-001/jetty1/index
   [junit4]   2> 253407 T1449 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@236389b8
   [junit4]   2> 253408 T1449 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be6a12e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2600225),segFN=segments_1,generation=1}
   [junit4]   2> 253408 T1449 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 253415 T1449 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 253415 T1449 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 253415 T1449 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 253416 T1449 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 253416 T1449 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 253416 T1449 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 253417 T1449 oasup.UpdateRequestProcessorChain.init creating updateR

[...truncated too long message...]

roxy.close WARN Closing 0 connections to: http://127.0.0.1:17435/jb_sfp/vb
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=95E8F46B89D049DF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Africa/Nouakchott -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   75.3s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17450/jb_sfp/vb/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([95E8F46B89D049DF:1DBCCBB1272C2427]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17450/jb_sfp/vb/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 1951031 T9452 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 95E8F46B89D049DF-001
   [junit4]   2> 75263 T9451 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=mk, timezone=Africa/Nouakchott
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=168675104,total=481820672
   [junit4]   2> NOTE: All tests run in this JVM: [ConvertedLegacyTest, TestSerializedLuceneMatchVersion, FieldAnalysisRequestHandlerTest, TestSimpleQParserPlugin, QueryElevationComponentTest, DistributedQueueTest, BufferStoreTest, VMParamsZkACLAndCredentialsProvidersTest, UpdateParamsTest, TestStressRecovery, IgnoreCommitOptimizeUpdateProcessorFactoryTest, RemoteQueryErrorTest, SystemInfoHandlerTest, CoreAdminCreateDiscoverTest, UniqFieldsUpdateProcessorFactoryTest, StressHdfsTest, JSONWriterTest, DistributedFacetPivotSmallTest, FacetPivotSmallTest, ZkControllerTest, DisMaxRequestHandlerTest, TestHighFrequencyDictionaryFactory, CopyFieldTest, CachingDirectoryFactoryTest, TestStressLucene, TestZkChroot, TestOmitPositions, PathHierarchyTokenizerFactoryTest, TestSweetSpotSimilarityFactory, DistributedQueryComponentCustomSortTest, TestCollapseQParserPlugin, TestLFUCache, TestAnalyzeInfixSuggestions, TestSearcherReuse, CursorMarkTest, TestPhraseSuggestions, LeaderFailoverAfterPartitionTest, TestInitQParser, TestSolrDeletionPolicy1, TestDocumentBuilder, AssignTest, PingRequestHandlerTest, BlockDirectoryTest, TestLMJelinekMercerSimilarityFactory, TestExpandComponent, TestFieldCollectionResource, TestSolr4Spatial, CircularListTest, WordBreakSolrSpellCheckerTest, TestFastOutputStream, OpenCloseCoreStressTest, TestReplicationHandlerBackup, DistributedTermsComponentTest, SSLMigrationTest, BadCopyFieldTest, TestSearchPerf, FullSolrCloudDistribCmdsTest, BasicZkTest, AutoCommitTest, TestDistributedGrouping, TestHashPartitioner, TestStressReorder, TestReload, StatsComponentTest, BadIndexSchemaTest, TestFiltering, TestBadConfig, TestIndexSearcher, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SuggesterFSTTest, TestFoldingMultitermQuery, SuggesterTest, SpatialFilterTest, PolyFieldTest, TestPseudoReturnFields, DefaultValueUpdateProcessorTest, SortByFunctionTest, DistanceFunctionTest, CacheHeaderTest, TestQueryUtils, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, XmlUpdateRequestHandlerTest, TermVectorComponentTest, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, IndexSchemaTest, TestCollationField, MBeansHandlerTest, TestAnalyzedSuggestions, CSVRequestHandlerTest, HighlighterConfigTest, AlternateDirectoryTest, SolrIndexConfigTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestCharFilters, EchoParamsTest, TestIBSimilarityFactory, TimeZoneUtilsTest, ScriptEngineTest, PluginInfoTest, PreAnalyzedFieldTest, DOMUtilTest, TestSolrJ, TestUtils, TestRTGBase, DistributedIntervalFacetingTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, OverriddenZkACLAndCredentialsProvidersTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.05s, 1 test, 1 error <<< FAILURES!

[...truncated 308 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1864 tests, 5 errors, 56 ignored (9 assumptions)

Total time: 146 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 669987559 bytes
Compression is 0.0%
Took 2 min 9 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 760 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/760/

6 tests failed.
FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:20657//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:20657//collection1
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:538)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:568)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:547)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33446/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33446/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33446/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13152/bi_/c8n_1x3_lf_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13152/bi_/c8n_1x3_lf_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13152/bi_/c8n_1x3_lf_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33578/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33578/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33008/_id/u/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33008/_id/u/collection1
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33008/_id/u/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33723/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33723/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33723/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 10528 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/init-core-data-001
   [junit4]   2> 1484173 T669706 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /bi_/
   [junit4]   2> 1484176 T669706 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1484177 T669707 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1484277 T669706 oasc.ZkTestServer.run start zk server on port:13149
   [junit4]   2> 1484278 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1484279 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1484282 T669714 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a64b841 name:ZooKeeperConnection Watcher:127.0.0.1:13149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1484282 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1484283 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1484283 T669706 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1484285 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1484287 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1484288 T669717 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c188b69 name:ZooKeeperConnection Watcher:127.0.0.1:13149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1484288 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1484289 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1484289 T669706 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1484291 T669706 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1484292 T669706 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1484293 T669706 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1484295 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1484295 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1484298 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1484298 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1484402 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1484403 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1484404 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1484405 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1484407 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1484407 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1484409 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1484409 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1484411 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1484412 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1484413 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1484414 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1484416 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1484416 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1484418 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1484419 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1484420 T669706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1484421 T669706 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1485637 T669706 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores/collection1
   [junit4]   2> 1485640 T669706 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1485646 T669706 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13153
   [junit4]   2> 1485646 T669706 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1485647 T669706 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1485647 T669706 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001
   [junit4]   2> 1485647 T669706 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/'
   [junit4]   2> 1485677 T669706 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/solr.xml
   [junit4]   2> 1485735 T669706 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores
   [junit4]   2> 1485736 T669706 oasc.CoreContainer.<init> New CoreContainer 259925002
   [junit4]   2> 1485736 T669706 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/]
   [junit4]   2> 1485737 T669706 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1485737 T669706 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1485737 T669706 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1485737 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1485738 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1485738 T669706 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1485738 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1485738 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1485738 T669706 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1485739 T669706 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1485739 T669706 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1485739 T669706 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1485740 T669706 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1485740 T669706 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1485740 T669706 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1485740 T669706 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13149/solr
   [junit4]   2> 1485740 T669706 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1485741 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1485741 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1485743 T669729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21185be2 name:ZooKeeperConnection Watcher:127.0.0.1:13149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1485744 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1485744 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1485746 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1485747 T669732 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4aabcb3b name:ZooKeeperConnection Watcher:127.0.0.1:13149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1485747 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1485749 T669706 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1485750 T669706 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1485752 T669706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1485754 T669706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1485756 T669706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1485757 T669706 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1485758 T669706 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13152_bi_
   [junit4]   2> 1485759 T669706 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13152_bi_
   [junit4]   2> 1485761 T669706 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1485762 T669706 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1485763 T669706 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1485764 T669706 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:13152_bi_
   [junit4]   2> 1485764 T669706 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1485765 T669706 oasc.Overseer.start Overseer (id=93301584290119683-127.0.0.1:13152_bi_-n_0000000000) starting
   [junit4]   2> 1485767 T669706 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1485772 T669706 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1485773 T669734 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1485774 T669706 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1485775 T669706 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1485776 T669706 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1485778 T669733 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1485779 T669706 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores
   [junit4]   2> 1485781 T669706 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores/collection1/
   [junit4]   2> 1485781 T669706 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1485782 T669736 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1485782 T669736 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1485783 T669736 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1485783 T669732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1485784 T669733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13152/bi_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13152_bi_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1485785 T669733 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13152/bi_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13152_bi_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1485785 T669733 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1485785 T669733 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1485786 T669732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1486783 T669736 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1486784 T669736 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1486784 T669736 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1486785 T669736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1486785 T669736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1486785 T669736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores/collection1/'
   [junit4]   2> 1486803 T669736 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1486805 T669736 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1486815 T669736 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1486837 T669736 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1486884 T669736 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1486986 T669736 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1486996 T669736 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1487380 T669736 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1487391 T669736 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1487393 T669736 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1487403 T669736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1487407 T669736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1487411 T669736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1487412 T669736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1487412 T669736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1487412 T669736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1487413 T669736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1487413 T669736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1487413 T669736 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1487414 T669736 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1487414 T669736 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/tempDir-001/control/data/
   [junit4]   2> 1487414 T669736 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6679ed9e
   [junit4]   2> 1487415 T669736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/tempDir-001/control/data
   [junit4]   2> 1487415 T669736 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/tempDir-001/control/data/index/
   [junit4]   2> 1487416 T669736 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1487416 T669736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/tempDir-001/control/data/index
   [junit4]   2> 1487416 T669736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=71.4921875, floorSegmentMB=1.0625, forceMergeDeletesPctAllowed=23.47952157076037, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5702338807853159
   [junit4]   2> 1487417 T669736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5003c1e3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26ae471f),segFN=segments_1,generation=1}
   [junit4]   2> 1487418 T669736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1487422 T669736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1487422 T669736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1487423 T669736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1487423 T669736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1487423 T669736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1487423 T669736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1487424 T669736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1487424 T669736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1487424 T669736 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1487425 T669736 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1487425 T669736 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1487426 T669736 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1487426 T669736 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1487427 T669736 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1487427 T669736 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1487428 T669736 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1487428 T669736 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1487429 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1487430 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1487430 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1487431 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1487431 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1487431 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1487432 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1487432 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1487433 T669736 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1487433 T669736 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1487452 T669736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1487454 T669736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1487456 T669736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1487457 T669736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1487459 T669736 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1487463 T669736 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1487463 T669736 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1487464 T669736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2563211583177789]
   [junit4]   2> 1487465 T669736 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5003c1e3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26ae471f),segFN=segments_1,generation=1}
   [junit4]   2> 1487465 T669736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1487465 T669736 oass.SolrIndexSearcher.<init> Opening Searcher@31494684[collection1] main
   [junit4]   2> 1487466 T669736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1487466 T669736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1487467 T669736 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1487467 T669736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1487467 T669736 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1487467 T669736 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1487468 T669736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1487468 T669736 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1487468 T669736 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1487469 T669736 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1487469 T669737 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@31494684[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1487470 T669736 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1487470 T669740 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13152/bi_ collection:control_collection shard:shard1
   [junit4]   2> 1487471 T669706 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 1487471 T669706 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1487471 T669740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1487473 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1487474 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1487474 T669740 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1487475 T669744 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b1996a name:ZooKeeperConnection Watcher:127.0.0.1:13149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1487475 T669732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1487476 T669740 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1487476 T669740 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1487476 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1487476 T669733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C31242 name=collection1 org.apache.solr.core.SolrCore@19204738 url=http://127.0.0.1:13152/bi_/collection1 node=127.0.0.1:13152_bi_ C31242_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:13152/bi_, core=collection1, node_name=127.0.0.1:13152_bi_}
   [junit4]   2> 1487476 T669740 C31242 P13152 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13152/bi_/collection1/
   [junit4]   2> 1487477 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1487477 T669740 C31242 P13152 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1487477 T669740 C31242 P13152 oasc.SyncStrategy.syncToMe http://127.0.0.1:13152/bi_/collection1/ has no replicas
   [junit4]   2> 1487477 T669732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1487478 T669706 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1487478 T669740 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13152/bi_/collection1/ shard1
   [junit4]   2> 1487478 T669740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1487480 T669706 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1487480 T669706 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1487480 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1487481 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1487481 T669747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1487482 T669748 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bee8c5d name:ZooKeeperConnection Watcher:127.0.0.1:13149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1487482 T669733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13152/bi_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1487482 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1487483 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1487484 T669747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1487486 T669733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1487486 T669733 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1487486 T669744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1487590 T669733 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1487591 T669744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1487633 T669740 oasc.ZkController.register We are http://127.0.0.1:13152/bi_/collection1/ and leader is http://127.0.0.1:13152/bi_/collection1/
   [junit4]   2> 1487634 T669740 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13152/bi_
   [junit4]   2> 1487634 T669740 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1487634 T669740 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1487634 T669740 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1487635 T669732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1487636 T669733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13152/bi_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "node_name":"127.0.0.1:13152_bi_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1487637 T669733 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13152/bi_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "node_name":"127.0.0.1:13152_bi_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1487740 T669732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1487740 T669744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1488724 T669706 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/cores/collection1
   [junit4]   2> 1488726 T669706 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1488726 T669706 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1488729 T669706 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13159
   [junit4]   2> 1488729 T669706 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1488729 T669706 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1488730 T669706 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001
   [junit4]   2> 1488730 T669706 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/'
   [junit4]   2> 1488759 T669706 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/solr.xml
   [junit4]   2> 1488792 T669706 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/cores
   [junit4]   2> 1488793 T669706 oasc.CoreContainer.<init> New CoreContainer 176332206
   [junit4]   2> 1488793 T669706 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/]
   [junit4]   2> 1488794 T669706 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1488794 T669706 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1488794 T669706 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1488794 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1488795 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1488795 T669706 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1488795 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1488795 T669706 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1488795 T669706 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1488796 T669706 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1488796 T669706 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1488797 T669706 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1488797 T669706 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1488797 T669706 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1488797 T669706 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1488797 T669706 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13149/solr
   [junit4]   2> 1488798 T669706 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1488798 T669706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1488799 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488800 T669760 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a28caa7 name:ZooKeeperConnection Watcher:127.0.0.1:13149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488801 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488802 T669706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1488804 T669706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488805 T669763 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60d9fed7 name:ZooKeeperConnection Watcher:127.0.0.1:13149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488805 T669706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488808 T669706 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1489812 T669706 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13158_bi_
   [junit4]   2> 1489813 T669706 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13158_bi_
   [junit4]   2> 1489816 T669706 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1489819 T669706 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 3B901651DA539072-001/shard-1-001/cores
   [junit4]   2> 1489821 T669706 oasc.Co

[...truncated too long message...]

3 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:33735/
   [junit4]   2> 2030108 T3843 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33730/
   [junit4]   2> 2030108 T3843 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:33723/
   [junit4]   2> 2030109 T3843 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:33709/
   [junit4]   2> 2030109 T3843 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2030109 T3843 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33735/
   [junit4]   2> 2030110 T3843 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33730/
   [junit4]   2> 2030110 T3843 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33723/
   [junit4]   2> 2030110 T3843 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33709/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=3B901651DA539072 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de -Dtests.timezone=Asia/Harbin -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   72.0s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33723/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3B901651DA539072:B3C4298B74AFFD8A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33723/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 2030123 T3843 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 3B901651DA539072-001
   [junit4]   2> 72035 T3842 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), _version_=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=Memory doPackFST= false), intDefault=BlockTreeOrds(blocksize=128), id=FSTOrd50, multiDefault=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=de, timezone=Asia/Harbin
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=144375576,total=332922880
   [junit4]   2> NOTE: All tests run in this JVM: [NotRequiredUniqueKeyTest, StatsComponentTest, SolrIndexSplitterTest, TestExactStatsCache, TestCursorMarkWithoutUniqueKey, TestManagedSynonymFilterFactory, SuggestComponentTest, QueryEqualityTest, TestFoldingMultitermQuery, SynonymTokenizerTest, FileBasedSpellCheckerTest, SimpleMLTQParserTest, DocumentBuilderTest, DirectUpdateHandlerTest, TestReload, TestLRUCache, TestManagedResource, TestStandardQParsers, StandardRequestHandlerTest, SpellCheckComponentTest, TestCloudManagedSchema, TestCopyFieldCollectionResource, TestOrdValues, TestNRTOpen, TestQuerySenderNoQuery, DistributedFacetPivotWhiteBoxTest, TestElisionMultitermQuery, OverriddenZkACLAndCredentialsProvidersTest, TestDefaultSearchFieldResource, TestSolrConfigHandlerConcurrent, SOLR749Test, TestRandomDVFaceting, TestClusterStateMutator, BasicDistributedZk2Test, TestFieldSortValues, AnalyticsQueryTest, TestLRUStatsCache, OverseerStatusTest, SliceStateTest, TestCloudInspectUtil, TestPerFieldSimilarity, LeaderElectionIntegrationTest, TestDocBasedVersionConstraints, SolrCoreCheckLockOnStartupTest, DateRangeFieldTest, TestReplicationHandler, NoCacheHeaderTest, RecoveryAfterSoftCommitTest, SharedFSAutoReplicaFailoverTest, ReturnFieldsTest, DateFieldTest, TestLazyCores, MultiThreadedOCPTest, TestOverriddenPrefixQueryForCustomFieldType, TestRemoteStreaming, TestReqParamsAPI, SolrXmlInZkTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 72.83s, 1 test, 1 error <<< FAILURES!

[...truncated 289 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1864 tests, 6 errors, 56 ignored (9 assumptions)

Total time: 146 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 451859035 bytes
Compression is 0.0%
Took 1 min 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 759 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/759/

7 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1423583446269,generation=2,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6o.fdt, _6o.fdx, _6o.fnm, _6o.nvd, _6o.nvm, _6o.si, _6o_LuceneFixedGap_0.doc, _6o_LuceneFixedGap_0.tib, _6o_LuceneFixedGap_0.tii, _6p.cfe, _6p.cfs, _6p.si, _6q.fdt, _6q.fdx, _6q.fnm, _6q.nvd, _6q.nvm, _6q.si, _6q_LuceneFixedGap_0.doc, _6q_LuceneFixedGap_0.tib, _6q_LuceneFixedGap_0.tii, _6r.fdt, _6r.fdx, _6r.fnm, _6r.nvd, _6r.nvm, _6r.si, _6r_LuceneFixedGap_0.doc, _6r_LuceneFixedGap_0.tib, _6r_LuceneFixedGap_0.tii, _6s.fdt, _6s.fdx, _6s.fnm, _6s.nvd, _6s.nvm, _6s.si, _6s_LuceneFixedGap_0.doc, _6s_LuceneFixedGap_0.tib, _6s_LuceneFixedGap_0.tii, _6t.fdt, _6t.fdx, _6t.fnm, _6t.nvd, _6t.nvm, _6t.si, _6t_LuceneFixedGap_0.doc, _6t_LuceneFixedGap_0.tib, _6t_LuceneFixedGap_0.tii, _6u.fdt, _6u.fdx, _6u.fnm, _6u.nvd, _6u.nvm, _6u.si, _6u_LuceneFixedGap_0.doc, _6u_LuceneFixedGap_0.tib, _6u_LuceneFixedGap_0.tii, _6v.fdt, _6v.fdx, _6v.fnm, _6v.nvd, _6v.nvm, _6v.si, _6v_LuceneFixedGap_0.doc, _6v_LuceneFixedGap_0.tib, _6v_LuceneFixedGap_0.tii, _6w.fdt, _6w.fdx, _6w.fnm, _6w.nvd, _6w.nvm, _6w.si, _6w_LuceneFixedGap_0.doc, _6w_LuceneFixedGap_0.tib, _6w_LuceneFixedGap_0.tii, _6x.fdt, _6x.fdx, _6x.fnm, _6x.nvd, _6x.nvm, _6x.si, _6x_LuceneFixedGap_0.doc, _6x_LuceneFixedGap_0.tib, _6x_LuceneFixedGap_0.tii, _6y.fdt, _6y.fdx, _6y.fnm, _6y.nvd, _6y.nvm, _6y.si, _6y_LuceneFixedGap_0.doc, _6y_LuceneFixedGap_0.tib, _6y_LuceneFixedGap_0.tii, _6z.fdt, _6z.fdx, _6z.fnm, _6z.nvd, _6z.nvm, _6z.si, _6z_LuceneFixedGap_0.doc, _6z_LuceneFixedGap_0.tib, _6z_LuceneFixedGap_0.tii, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_LuceneFixedGap_0.doc, _70_LuceneFixedGap_0.tib, _70_LuceneFixedGap_0.tii, _71.fdt, _71.fdx, _71.fnm, _71.nvd, _71.nvm, _71.si, _71_LuceneFixedGap_0.doc, _71_LuceneFixedGap_0.tib, _71_LuceneFixedGap_0.tii, _72.fdt, _72.fdx, _72.fnm, _72.nvd, _72.nvm, _72.si, _72_LuceneFixedGap_0.doc, _72_LuceneFixedGap_0.tib, _72_LuceneFixedGap_0.tii, segments_2]}]> but was:<[{indexVersion=1423583446269,generation=3,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6p.cfe, _6p.cfs, _6p.si, _73.cfe, _73.cfs, _73.si, segments_3]}, {indexVersion=1423583446269,generation=2,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6o.fdt, _6o.fdx, _6o.fnm, _6o.nvd, _6o.nvm, _6o.si, _6o_LuceneFixedGap_0.doc, _6o_LuceneFixedGap_0.tib, _6o_LuceneFixedGap_0.tii, _6p.cfe, _6p.cfs, _6p.si, _6q.fdt, _6q.fdx, _6q.fnm, _6q.nvd, _6q.nvm, _6q.si, _6q_LuceneFixedGap_0.doc, _6q_LuceneFixedGap_0.tib, _6q_LuceneFixedGap_0.tii, _6r.fdt, _6r.fdx, _6r.fnm, _6r.nvd, _6r.nvm, _6r.si, _6r_LuceneFixedGap_0.doc, _6r_LuceneFixedGap_0.tib, _6r_LuceneFixedGap_0.tii, _6s.fdt, _6s.fdx, _6s.fnm, _6s.nvd, _6s.nvm, _6s.si, _6s_LuceneFixedGap_0.doc, _6s_LuceneFixedGap_0.tib, _6s_LuceneFixedGap_0.tii, _6t.fdt, _6t.fdx, _6t.fnm, _6t.nvd, _6t.nvm, _6t.si, _6t_LuceneFixedGap_0.doc, _6t_LuceneFixedGap_0.tib, _6t_LuceneFixedGap_0.tii, _6u.fdt, _6u.fdx, _6u.fnm, _6u.nvd, _6u.nvm, _6u.si, _6u_LuceneFixedGap_0.doc, _6u_LuceneFixedGap_0.tib, _6u_LuceneFixedGap_0.tii, _6v.fdt, _6v.fdx, _6v.fnm, _6v.nvd, _6v.nvm, _6v.si, _6v_LuceneFixedGap_0.doc, _6v_LuceneFixedGap_0.tib, _6v_LuceneFixedGap_0.tii, _6w.fdt, _6w.fdx, _6w.fnm, _6w.nvd, _6w.nvm, _6w.si, _6w_LuceneFixedGap_0.doc, _6w_LuceneFixedGap_0.tib, _6w_LuceneFixedGap_0.tii, _6x.fdt, _6x.fdx, _6x.fnm, _6x.nvd, _6x.nvm, _6x.si, _6x_LuceneFixedGap_0.doc, _6x_LuceneFixedGap_0.tib, _6x_LuceneFixedGap_0.tii, _6y.fdt, _6y.fdx, _6y.fnm, _6y.nvd, _6y.nvm, _6y.si, _6y_LuceneFixedGap_0.doc, _6y_LuceneFixedGap_0.tib, _6y_LuceneFixedGap_0.tii, _6z.fdt, _6z.fdx, _6z.fnm, _6z.nvd, _6z.nvm, _6z.si, _6z_LuceneFixedGap_0.doc, _6z_LuceneFixedGap_0.tib, _6z_LuceneFixedGap_0.tii, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_LuceneFixedGap_0.doc, _70_LuceneFixedGap_0.tib, _70_LuceneFixedGap_0.tii, _71.fdt, _71.fdx, _71.fnm, _71.nvd, _71.nvm, _71.si, _71_LuceneFixedGap_0.doc, _71_LuceneFixedGap_0.tib, _71_LuceneFixedGap_0.tii, _72.fdt, _72.fdx, _72.fnm, _72.nvd, _72.nvm, _72.si, _72_LuceneFixedGap_0.doc, _72_LuceneFixedGap_0.tib, _72_LuceneFixedGap_0.tii, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1423583446269,generation=2,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6o.fdt, _6o.fdx, _6o.fnm, _6o.nvd, _6o.nvm, _6o.si, _6o_LuceneFixedGap_0.doc, _6o_LuceneFixedGap_0.tib, _6o_LuceneFixedGap_0.tii, _6p.cfe, _6p.cfs, _6p.si, _6q.fdt, _6q.fdx, _6q.fnm, _6q.nvd, _6q.nvm, _6q.si, _6q_LuceneFixedGap_0.doc, _6q_LuceneFixedGap_0.tib, _6q_LuceneFixedGap_0.tii, _6r.fdt, _6r.fdx, _6r.fnm, _6r.nvd, _6r.nvm, _6r.si, _6r_LuceneFixedGap_0.doc, _6r_LuceneFixedGap_0.tib, _6r_LuceneFixedGap_0.tii, _6s.fdt, _6s.fdx, _6s.fnm, _6s.nvd, _6s.nvm, _6s.si, _6s_LuceneFixedGap_0.doc, _6s_LuceneFixedGap_0.tib, _6s_LuceneFixedGap_0.tii, _6t.fdt, _6t.fdx, _6t.fnm, _6t.nvd, _6t.nvm, _6t.si, _6t_LuceneFixedGap_0.doc, _6t_LuceneFixedGap_0.tib, _6t_LuceneFixedGap_0.tii, _6u.fdt, _6u.fdx, _6u.fnm, _6u.nvd, _6u.nvm, _6u.si, _6u_LuceneFixedGap_0.doc, _6u_LuceneFixedGap_0.tib, _6u_LuceneFixedGap_0.tii, _6v.fdt, _6v.fdx, _6v.fnm, _6v.nvd, _6v.nvm, _6v.si, _6v_LuceneFixedGap_0.doc, _6v_LuceneFixedGap_0.tib, _6v_LuceneFixedGap_0.tii, _6w.fdt, _6w.fdx, _6w.fnm, _6w.nvd, _6w.nvm, _6w.si, _6w_LuceneFixedGap_0.doc, _6w_LuceneFixedGap_0.tib, _6w_LuceneFixedGap_0.tii, _6x.fdt, _6x.fdx, _6x.fnm, _6x.nvd, _6x.nvm, _6x.si, _6x_LuceneFixedGap_0.doc, _6x_LuceneFixedGap_0.tib, _6x_LuceneFixedGap_0.tii, _6y.fdt, _6y.fdx, _6y.fnm, _6y.nvd, _6y.nvm, _6y.si, _6y_LuceneFixedGap_0.doc, _6y_LuceneFixedGap_0.tib, _6y_LuceneFixedGap_0.tii, _6z.fdt, _6z.fdx, _6z.fnm, _6z.nvd, _6z.nvm, _6z.si, _6z_LuceneFixedGap_0.doc, _6z_LuceneFixedGap_0.tib, _6z_LuceneFixedGap_0.tii, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_LuceneFixedGap_0.doc, _70_LuceneFixedGap_0.tib, _70_LuceneFixedGap_0.tii, _71.fdt, _71.fdx, _71.fnm, _71.nvd, _71.nvm, _71.si, _71_LuceneFixedGap_0.doc, _71_LuceneFixedGap_0.tib, _71_LuceneFixedGap_0.tii, _72.fdt, _72.fdx, _72.fnm, _72.nvd, _72.nvm, _72.si, _72_LuceneFixedGap_0.doc, _72_LuceneFixedGap_0.tib, _72_LuceneFixedGap_0.tii, segments_2]}]> but was:<[{indexVersion=1423583446269,generation=3,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6p.cfe, _6p.cfs, _6p.si, _73.cfe, _73.cfs, _73.si, segments_3]}, {indexVersion=1423583446269,generation=2,filelist=[_5v.fdt, _5v.fdx, _5v.fnm, _5v.nvd, _5v.nvm, _5v.si, _5v_LuceneFixedGap_0.doc, _5v_LuceneFixedGap_0.tib, _5v_LuceneFixedGap_0.tii, _6a.cfe, _6a.cfs, _6a.si, _6o.fdt, _6o.fdx, _6o.fnm, _6o.nvd, _6o.nvm, _6o.si, _6o_LuceneFixedGap_0.doc, _6o_LuceneFixedGap_0.tib, _6o_LuceneFixedGap_0.tii, _6p.cfe, _6p.cfs, _6p.si, _6q.fdt, _6q.fdx, _6q.fnm, _6q.nvd, _6q.nvm, _6q.si, _6q_LuceneFixedGap_0.doc, _6q_LuceneFixedGap_0.tib, _6q_LuceneFixedGap_0.tii, _6r.fdt, _6r.fdx, _6r.fnm, _6r.nvd, _6r.nvm, _6r.si, _6r_LuceneFixedGap_0.doc, _6r_LuceneFixedGap_0.tib, _6r_LuceneFixedGap_0.tii, _6s.fdt, _6s.fdx, _6s.fnm, _6s.nvd, _6s.nvm, _6s.si, _6s_LuceneFixedGap_0.doc, _6s_LuceneFixedGap_0.tib, _6s_LuceneFixedGap_0.tii, _6t.fdt, _6t.fdx, _6t.fnm, _6t.nvd, _6t.nvm, _6t.si, _6t_LuceneFixedGap_0.doc, _6t_LuceneFixedGap_0.tib, _6t_LuceneFixedGap_0.tii, _6u.fdt, _6u.fdx, _6u.fnm, _6u.nvd, _6u.nvm, _6u.si, _6u_LuceneFixedGap_0.doc, _6u_LuceneFixedGap_0.tib, _6u_LuceneFixedGap_0.tii, _6v.fdt, _6v.fdx, _6v.fnm, _6v.nvd, _6v.nvm, _6v.si, _6v_LuceneFixedGap_0.doc, _6v_LuceneFixedGap_0.tib, _6v_LuceneFixedGap_0.tii, _6w.fdt, _6w.fdx, _6w.fnm, _6w.nvd, _6w.nvm, _6w.si, _6w_LuceneFixedGap_0.doc, _6w_LuceneFixedGap_0.tib, _6w_LuceneFixedGap_0.tii, _6x.fdt, _6x.fdx, _6x.fnm, _6x.nvd, _6x.nvm, _6x.si, _6x_LuceneFixedGap_0.doc, _6x_LuceneFixedGap_0.tib, _6x_LuceneFixedGap_0.tii, _6y.fdt, _6y.fdx, _6y.fnm, _6y.nvd, _6y.nvm, _6y.si, _6y_LuceneFixedGap_0.doc, _6y_LuceneFixedGap_0.tib, _6y_LuceneFixedGap_0.tii, _6z.fdt, _6z.fdx, _6z.fnm, _6z.nvd, _6z.nvm, _6z.si, _6z_LuceneFixedGap_0.doc, _6z_LuceneFixedGap_0.tib, _6z_LuceneFixedGap_0.tii, _70.fdt, _70.fdx, _70.fnm, _70.nvd, _70.nvm, _70.si, _70_LuceneFixedGap_0.doc, _70_LuceneFixedGap_0.tib, _70_LuceneFixedGap_0.tii, _71.fdt, _71.fdx, _71.fnm, _71.nvd, _71.nvm, _71.si, _71_LuceneFixedGap_0.doc, _71_LuceneFixedGap_0.tib, _71_LuceneFixedGap_0.tii, _72.fdt, _72.fdx, _72.fnm, _72.nvd, _72.nvm, _72.si, _72_LuceneFixedGap_0.doc, _72_LuceneFixedGap_0.tib, _72_LuceneFixedGap_0.tii, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:542F538AF8581A6C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:62000//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:62000//collection1
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:538)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:568)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:547)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:60503/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60503/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10585 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/init-core-data-001
   [junit4]   2> 163358 T450 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 163363 T450 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 163364 T451 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 163464 T450 oasc.ZkTestServer.run start zk server on port:27107
   [junit4]   2> 163465 T450 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 163466 T450 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 163470 T458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a97b7f5 name:ZooKeeperConnection Watcher:127.0.0.1:27107 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 163470 T450 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 163471 T450 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 163471 T450 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 163475 T450 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 163476 T450 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 163478 T461 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e817b22 name:ZooKeeperConnection Watcher:127.0.0.1:27107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 163478 T450 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 163479 T450 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 163479 T450 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 163482 T450 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 163483 T450 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 163485 T450 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 163487 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 163488 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 163491 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 163492 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 163494 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 163495 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 163498 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 163498 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 163501 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 163501 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 163503 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 163504 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 163506 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 163507 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 163509 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 163509 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 163511 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 163512 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 163514 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 163515 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 163517 T450 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 163517 T450 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 165026 T450 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores/collection1
   [junit4]   2> 165031 T450 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 165035 T450 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27111
   [junit4]   2> 165036 T450 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 165036 T450 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 165037 T450 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001
   [junit4]   2> 165037 T450 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/'
   [junit4]   2> 165084 T450 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/solr.xml
   [junit4]   2> 165137 T450 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores
   [junit4]   2> 165138 T450 oasc.CoreContainer.<init> New CoreContainer 1060604517
   [junit4]   2> 165138 T450 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/]
   [junit4]   2> 165140 T450 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 165140 T450 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 165140 T450 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 165141 T450 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 165141 T450 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 165141 T450 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 165141 T450 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 165142 T450 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 165142 T450 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 165142 T450 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 165143 T450 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 165143 T450 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 165144 T450 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 165144 T450 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 165144 T450 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 165145 T450 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27107/solr
   [junit4]   2> 165146 T450 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 165146 T450 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 165147 T450 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 165149 T473 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35a29e7f name:ZooKeeperConnection Watcher:127.0.0.1:27107 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 165150 T450 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 165151 T450 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 165153 T450 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 165155 T476 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f0b8593 name:ZooKeeperConnection Watcher:127.0.0.1:27107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 165155 T450 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 165157 T450 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 165160 T450 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 165162 T450 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 165165 T450 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 165167 T450 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 165169 T450 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 165171 T450 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27110_
   [junit4]   2> 165171 T450 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27110_
   [junit4]   2> 165173 T450 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 165175 T450 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 165176 T450 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 165178 T450 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:27110_
   [junit4]   2> 165179 T450 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 165180 T450 oasc.Overseer.start Overseer (id=93295889640259587-127.0.0.1:27110_-n_0000000000) starting
   [junit4]   2> 165182 T450 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 165189 T450 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 165190 T478 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 165191 T450 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 165193 T450 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 165194 T450 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 165197 T477 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 165199 T450 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores
   [junit4]   2> 165201 T450 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores/collection1/
   [junit4]   2> 165202 T450 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 165203 T480 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 165203 T480 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 165204 T480 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 165205 T476 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 165207 T477 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27110_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27110"} current state version: 0
   [junit4]   2> 165208 T477 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27110_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27110"}
   [junit4]   2> 165208 T477 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 165208 T477 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 165210 T476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 166205 T480 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 166206 T480 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 166206 T480 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 166207 T480 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 166207 T480 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 166208 T480 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores/collection1/'
   [junit4]   2> 166226 T480 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 166229 T480 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 166244 T480 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 166267 T480 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 166318 T480 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 166319 T480 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 166331 T480 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 166738 T480 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 166750 T480 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 166753 T480 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 166766 T480 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166771 T480 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166776 T480 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166777 T480 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166777 T480 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 166777 T480 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166778 T480 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166778 T480 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 166778 T480 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 166779 T480 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 166779 T480 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/tempDir-001/control/data/
   [junit4]   2> 166779 T480 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@51148f22
   [junit4]   2> 166780 T480 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/tempDir-001/control/data
   [junit4]   2> 166780 T480 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/tempDir-001/control/data/index/
   [junit4]   2> 166781 T480 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 166790 T480 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/tempDir-001/control/data/index
   [junit4]   2> 166790 T480 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 166796 T480 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@65e719a0),segFN=segments_1,generation=1}
   [junit4]   2> 166796 T480 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 166802 T480 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 166802 T480 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 166803 T480 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 166803 T480 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 166803 T480 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 166804 T480 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 166804 T480 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 166804 T480 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 166805 T480 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 166805 T480 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 166806 T480 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 166806 T480 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 166807 T480 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 166807 T480 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 166808 T480 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 166808 T480 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 166809 T480 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 166809 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 166810 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 166811 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 166811 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 166812 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 166812 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 166813 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 166813 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 166814 T480 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 166814 T480 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 166835 T480 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 166838 T480 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 166840 T480 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 166842 T480 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 166844 T480 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 166849 T480 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 166849 T480 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 166850 T480 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=7, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.9384765625, noCFSRatio=0.6125000433057339]
   [junit4]   2> 166854 T480 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 71F848BA8810146F-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@65e719a0),segFN=segments_1,generation=1}
   [junit4]   2> 166854 T480 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 166855 T480 oass.SolrIndexSearcher.<init> Opening Searcher@2c25501c[collection1] main
   [junit4]   2> 166855 T480 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 166856 T480 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 166856 T480 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 166856 T480 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 166857 T480 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 166857 T480 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 166857 T480 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 166858 T480 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 166858 T480 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 166858 T480 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 166859 T481 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2c25501c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 166860 T480 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 166861 T484 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27110 collection:control_collection shard:shard1
   [junit4]   2> 166861 T450 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 166862 T450 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 166862 T484 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 166865 T450 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166866 T450 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166867 T484 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 166868 T488 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66e536a3 name:ZooKeeperConnection Watcher:127.0.0.1:27107/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166868 T450 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166869 T476 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 166869 T450 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166869 T484 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 166870 T484 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C114 name=collection1 org.apache.solr.core.SolrCore@787e88e url=http://127.0.0.1:27110/collection1 node=127.0.0.1:27110_ C114_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:27110_, state=down, base_url=http://127.0.0.1:27110}
   [junit4]   2> 166870 T484 C114 P27110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27110/collection1/
   [junit4]   2> 166870 T450 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 166870 T477 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 166870 T484 C114 P27110 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 166871 T484 C114 P27110 oasc.SyncStrategy.syncToMe http://127.0.0.1:27110/collection1/ has no replicas
   [junit4]   2> 166871 T484 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27110/collection1/ shard1
   [junit4]   2> 166871 T484 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 166872 T488 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 166872 T476 oascc.ZkStateReader$2.process A cluster state change: WatchedEven

[...truncated too long message...]

lucene-data/enwiki.random.lines.txt -Dtests.locale=es_GT -Dtests.timezone=Africa/Sao_Tome -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   62.7s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([71F848BA8810146F:F9AC776026EC7997]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1970200 T6981 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 71F848BA8810146F-001
   [junit4]   2> 62734 T6980 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {text=FSTOrd50, id=PostingsFormat(name=Memory doPackFST= false), _version_=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct), intDefault=Lucene50(blocksize=128), a_t=PostingsFormat(name=Direct)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=es_GT, timezone=Africa/Sao_Tome
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=143421536,total=333447168
   [junit4]   2> NOTE: All tests run in this JVM: [TestHighlightDedupGrouping, TestTrie, SpellCheckCollatorTest, HdfsLockFactoryTest, ZkNodePropsTest, TestIBSimilarityFactory, UUIDFieldTest, PreAnalyzedUpdateProcessorTest, ShowFileRequestHandlerTest, TestAddFieldRealTimeGet, IndexBasedSpellCheckerTest, OverseerRolesTest, TestStressReorder, ReplicationFactorTest, DocExpirationUpdateProcessorFactoryTest, TestDistribDocBasedVersion, TestUpdate, DOMUtilTest, TestCoreContainer, ZkSolrClientTest, TestCharFilters, PrimUtilsTest, TestSolrCoreProperties, TestSortingResponseWriter, TestSolrDeletionPolicy2, TestImplicitCoreProperties, BasicDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest, OverseerCollectionProcessorTest, SpatialRPTFieldTypeTest, TestRandomMergePolicy, TestMiniSolrCloudClusterSSL, BlockCacheTest, HdfsCollectionsAPIDistributedZkTest, TestSchemaNameResource, TestCollationField, SignatureUpdateProcessorFactoryTest, DistributedSpellCheckComponentTest, MBeansHandlerTest, DistributedQueryElevationComponentTest, DistributedDebugComponentTest, TestSolrJ, TestSort, OpenExchangeRatesOrgProviderTest, EnumFieldTest, TestRestManager, PrimitiveFieldTypeTest, BasicFunctionalityTest, TestSolrIndexConfig, TestBulkSchemaConcurrent, AliasIntegrationTest, TestXIncludeConfig, MinimalSchemaTest, TimeZoneUtilsTest, XmlUpdateRequestHandlerTest, HdfsDirectoryFactoryTest, TriLevelCompositeIdRoutingTest, AsyncMigrateRouteKeyTest, OverseerTest, LeaderElectionIntegrationTest, BasicZkTest, ClusterStateUpdateTest, TestRandomFaceting, SolrCmdDistributorTest, BadIndexSchemaTest, ConvertedLegacyTest, TestLazyCores, CoreAdminHandlerTest, TestStressLucene, TestCSVLoader, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, SolrInfoMBeanTest, TestQueryUtils, TestWriterPerf, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest, IndexSchemaTest, ReturnFieldsTest, UpdateRequestProcessorFactoryTest, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestComponentsName, TestLFUCache, SolrIndexConfigTest, TestDocSet, TestSearchPerf, TestFuzzyAnalyzedSuggestions, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SynonymTokenizerTest, TestDFRSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestJmxMonitoredMap, URLClassifyProcessorTest, DateMathParserTest, PreAnalyzedFieldTest, DateFieldTest, RAMDirectoryFactoryTest, ClusterStateTest, SliceStateTest, SystemInfoHandlerTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestTolerantSearch, AssignTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, CustomCollectionTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 63.51s, 1 test, 1 error <<< FAILURES!

[...truncated 300 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 466 suites, 1861 tests, 6 errors, 1 failure, 56 ignored (9 assumptions)

Total time: 162 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 710722456 bytes
Compression is 0.0%
Took 2 min 14 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 758 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/758/

7 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
shard4 is not consistent.  Got 324 from http://127.0.0.1:59280/yo/bk/collection1lastClient and got 297 from http://127.0.0.1:59362/yo/bk/collection1

Stack Trace:
java.lang.AssertionError: shard4 is not consistent.  Got 324 from http://127.0.0.1:59280/yo/bk/collection1lastClient and got 297 from http://127.0.0.1:59362/yo/bk/collection1
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1246)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1225)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.search.TestSearcherReuse.test

Error Message:
expected same:<Searcher@7207e854[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c3) Uninverting(_1(5.1.0):c1)))}> was not:<Searcher@389e903c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.1.0):c4)))}>

Stack Trace:
java.lang.AssertionError: expected same:<Searcher@7207e854[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c3) Uninverting(_1(5.1.0):c1)))}> was not:<Searcher@389e903c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.1.0):c4)))}>
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.junit.Assert.assertSame(Assert.java:593)
	at org.apache.solr.search.TestSearcherReuse.assertSearcherHasNotChanged(TestSearcherReuse.java:247)
	at org.apache.solr.search.TestSearcherReuse.test(TestSearcherReuse.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59524/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59524/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10025 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/init-core-data-001
   [junit4]   2> 530025 T2934 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /i_vcr/
   [junit4]   2> 530030 T2934 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 530031 T2935 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 530131 T2934 oasc.ZkTestServer.run start zk server on port:35255
   [junit4]   2> 530132 T2934 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 530133 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 530137 T2942 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d31f5b name:ZooKeeperConnection Watcher:127.0.0.1:35255 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 530137 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 530138 T2934 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 530138 T2934 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 530141 T2934 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 530142 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 530144 T2945 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64727568 name:ZooKeeperConnection Watcher:127.0.0.1:35255/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 530144 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 530144 T2934 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 530145 T2934 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 530147 T2934 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 530149 T2934 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 530150 T2934 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 530152 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 530153 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 530156 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 530157 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 530159 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 530160 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 530162 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 530162 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 530164 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 530165 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 530167 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 530167 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 530169 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 530170 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 530172 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 530173 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 530175 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 530176 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 530183 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 530184 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 530186 T2934 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 530187 T2934 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 532068 T2934 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores/collection1
   [junit4]   2> 532073 T2934 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 532081 T2934 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35259
   [junit4]   2> 532082 T2934 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 532082 T2934 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 532083 T2934 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001
   [junit4]   2> 532083 T2934 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/'
   [junit4]   2> 532149 T2934 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/solr.xml
   [junit4]   2> 532209 T2934 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores
   [junit4]   2> 532210 T2934 oasc.CoreContainer.<init> New CoreContainer 1634767501
   [junit4]   2> 532210 T2934 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/]
   [junit4]   2> 532212 T2934 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 532212 T2934 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 532213 T2934 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 532213 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 532213 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 532214 T2934 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 532214 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 532214 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 532215 T2934 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 532215 T2934 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 532215 T2934 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 532216 T2934 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 532216 T2934 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 532217 T2934 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 532217 T2934 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 532218 T2934 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35255/solr
   [junit4]   2> 532218 T2934 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 532218 T2934 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 532219 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 532221 T2957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c570754 name:ZooKeeperConnection Watcher:127.0.0.1:35255 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 532222 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 532223 T2934 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 532225 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 532227 T2960 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67076126 name:ZooKeeperConnection Watcher:127.0.0.1:35255/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 532227 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 532229 T2934 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 532231 T2934 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 532233 T2934 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 532236 T2934 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 532238 T2934 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 532240 T2934 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 532242 T2934 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35258_i_vcr
   [junit4]   2> 532242 T2934 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35258_i_vcr
   [junit4]   2> 532245 T2934 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 532246 T2934 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 532248 T2934 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 532250 T2934 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:35258_i_vcr
   [junit4]   2> 532250 T2934 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 532252 T2934 oasc.Overseer.start Overseer (id=93290344004648963-127.0.0.1:35258_i_vcr-n_0000000000) starting
   [junit4]   2> 532254 T2934 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 532261 T2934 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 532262 T2962 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 532262 T2934 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 532264 T2934 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 532265 T2934 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 532268 T2961 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 532270 T2934 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores
   [junit4]   2> 532272 T2934 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores/collection1/
   [junit4]   2> 532273 T2934 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 532274 T2964 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 532274 T2964 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 532275 T2964 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 532275 T2960 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 532278 T2961 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35258_i_vcr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35258/i_vcr",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 532278 T2961 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35258_i_vcr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35258/i_vcr",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 532279 T2961 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 532279 T2961 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 532280 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 533276 T2964 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 533276 T2964 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 533277 T2964 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 533277 T2964 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 533278 T2964 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 533278 T2964 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores/collection1/'
   [junit4]   2> 533299 T2964 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 533302 T2964 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 533316 T2964 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 533342 T2964 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 533394 T2964 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 533395 T2964 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 533407 T2964 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 533848 T2964 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 533861 T2964 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 533864 T2964 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 533876 T2964 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 533880 T2964 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 533885 T2964 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 533886 T2964 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 533887 T2964 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 533887 T2964 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 533888 T2964 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 533888 T2964 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 533888 T2964 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 533889 T2964 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 533889 T2964 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/tempDir-001/control/data/
   [junit4]   2> 533889 T2964 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@a18d964
   [junit4]   2> 533890 T2964 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/tempDir-001/control/data
   [junit4]   2> 533891 T2964 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/tempDir-001/control/data/index/
   [junit4]   2> 533891 T2964 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 533891 T2964 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/tempDir-001/control/data/index
   [junit4]   2> 533892 T2964 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@40d5b5aa
   [junit4]   2> 533893 T2964 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e89cd27 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@78fbc2e5),segFN=segments_1,generation=1}
   [junit4]   2> 533893 T2964 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 533898 T2964 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 533898 T2964 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 533899 T2964 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 533899 T2964 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 533899 T2964 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 533899 T2964 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 533900 T2964 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 533900 T2964 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 533901 T2964 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 533901 T2964 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 533902 T2964 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 533902 T2964 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 533903 T2964 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 533904 T2964 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 533904 T2964 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 533905 T2964 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 533905 T2964 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 533906 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 533907 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 533908 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 533908 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 533909 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 533909 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 533910 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 533911 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 533911 T2964 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 533912 T2964 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 533933 T2964 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 533936 T2964 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 533938 T2964 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 533940 T2964 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 533942 T2964 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 533946 T2964 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 533946 T2964 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 533947 T2964 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1d8f57e0
   [junit4]   2> 533948 T2964 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e89cd27 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@78fbc2e5),segFN=segments_1,generation=1}
   [junit4]   2> 533948 T2964 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 533949 T2964 oass.SolrIndexSearcher.<init> Opening Searcher@91ae13[collection1] main
   [junit4]   2> 533949 T2964 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 533950 T2964 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 533951 T2964 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 533951 T2964 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 533951 T2964 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 533952 T2964 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 533952 T2964 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 533953 T2964 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 533953 T2964 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 533953 T2964 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 533955 T2965 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@91ae13[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 533955 T2964 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 533956 T2968 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:35258/i_vcr collection:control_collection shard:shard1
   [junit4]   2> 533956 T2934 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 533957 T2934 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 533957 T2968 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 533961 T2934 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 533962 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 533962 T2968 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 533964 T2972 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b38e967 name:ZooKeeperConnection Watcher:127.0.0.1:35255/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 533964 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 533964 T2960 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 533965 T2968 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 533965 T2968 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 533965 T2934 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 533965 T2961 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C420 name=collection1 org.apache.solr.core.SolrCore@66e7b13d url=http://127.0.0.1:35258/i_vcr/collection1 node=127.0.0.1:35258_i_vcr C420_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:35258_i_vcr, base_url=http://127.0.0.1:35258/i_vcr, core=collection1, state=down}
   [junit4]   2> 533965 T2968 C420 P35258 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:35258/i_vcr/collection1/
   [junit4]   2> 533966 T2968 C420 P35258 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 533967 T2934 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 533967 T2968 C420 P35258 oasc.SyncStrategy.syncToMe http://127.0.0.1:35258/i_vcr/collection1/ has no replicas
   [junit4]   2> 533967 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 533967 T2968 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:35258/i_vcr/collection1/ shard1
   [junit4]   2> 533968 T2968 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 533970 T2934 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 533972 T2973 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 533973 T2961 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35258/i_vcr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 534077 T2972 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 534125 T2968 oasc.ZkController.register We are http://127.0.0.1:35258/i_vcr/collection1/ and leader is http://127.0.0.1:35258/i_vcr/collection1/
   [junit4]   2> 534126 T2968 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:35258/i_vcr
   [junit4]   2> 534126 T2968 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 534126 T2968 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 534126 T2968 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 534128 T2960 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 534129 T2961 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:35258_i_vcr",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35258/i_vcr",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 534130 T2961 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:35258_i_vcr",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35258/i_vcr",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 534233 T2960 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 534233 T2972 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 535761 T2934 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/cores/collection1
   [junit4]   2> 535764 T2934 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 535765 T2934 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 535768 T2934 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35264
   [junit4]   2> 535769 T2934 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 535769 T2934 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 535770 T2934 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001
   [junit4]   2> 535770 T2934 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/'
   [junit4]   2> 535826 T2934 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/solr.xml
   [junit4]   2> 535888 T2934 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/cores
   [junit4]   2> 535889 T2934 oasc.CoreContainer.<init> New CoreContainer 1175242620
   [junit4]   2> 535889 T2934 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/]
   [junit4]   2> 535891 T2934 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 535891 T2934 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 535895 T2934 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 535895 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 535895 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 535896 T2934 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 535896 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 535896 T2934 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 535897 T2934 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 535897 T2934 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 535897 T2934 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 535898 T2934 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 535899 T2934 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 535899 T2934 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 535899 T2934 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 535900 T2934 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35255/solr
   [junit4]   2> 535900 T2934 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 535900 T2934 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 535901 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535903 T2985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@298d53b2 name:ZooKeeperConnection Watcher:127.0.0.1:35255 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535904 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535905 T2934 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 535907 T2934 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535908 T2988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@82ca42e name:ZooKeeperConnection Watcher:127.0.0.1:35255/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535909 T2934 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535915 T2934 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 536919 T2934 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35263_i_vcr
   [junit4]   2> 536920 T2934 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35263_i_vcr
   [junit4]   2> 536924 T2934 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 536927 T2934 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/cores
   [junit4]   2> 536929 T2934 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B33638437AC7E87F-001/shard-1-001/cores/collection1/
   [junit4]   2> 536930 T2934 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 536931 T2989 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 536931 T2989 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 536932 T2989 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 536932 T2960 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 536933 T2961 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35263_i_vcr",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35263/i_vcr",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 536934 T2961 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:35263_i_vcr",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35263/i_vcr",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 536935 T2961 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 536935 T2961 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 53693

[...truncated too long message...]

e,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2403685 T8541 C2855 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=NRTCachingDirectory(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001/index lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2103fa76; maxCacheMB=48.0 maxMergeSizeMB=4.0),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=NRTCachingDirectory(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001/index lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2103fa76; maxCacheMB=48.0 maxMergeSizeMB=4.0),segFN=segments_5,generation=5}
   [junit4]   2> 2403686 T8541 C2855 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 2403687 T8541 C2855 oass.SolrIndexSearcher.<init> Opening Searcher@68ecaa96[collection1] main
   [junit4]   2> 2403687 T8543 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68ecaa96[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2403687 T8541 C2855 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 2403688 T8541 C2855 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 6
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherReuse -Dtests.method=test -Dtests.seed=B33638437AC7E87F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=sr_RS -Dtests.timezone=Europe/Prague -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.11s J2 | TestSearcherReuse.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected same:<Searcher@7207e854[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c3) Uninverting(_1(5.1.0):c1)))}> was not:<Searcher@389e903c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.1.0):c4)))}>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B33638437AC7E87F:3B620799D43B8587]:0)
   [junit4]    > 	at org.apache.solr.search.TestSearcherReuse.assertSearcherHasNotChanged(TestSearcherReuse.java:247)
   [junit4]    > 	at org.apache.solr.search.TestSearcherReuse.test(TestSearcherReuse.java:104)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2403697 T8541 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 2403698 T8541 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1256806055
   [junit4]   2> 2403698 T8541 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@50845ced
   [junit4]   2> 2403699 T8541 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=4,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=329,transaction_logs_total_number=3}
   [junit4]   2> 2403699 T8541 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2403699 T8541 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2403699 T8541 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2403700 T8541 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2403701 T8541 oasc.CachingDirectoryFactory.close Closing NRTCachingDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2403701 T8541 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 2403701 T8541 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001/index
   [junit4]   2> 2403701 T8541 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001;done=false>>]
   [junit4]   2> 2403702 T8541 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.TestSearcherReuse B33638437AC7E87F-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=sr_RS, timezone=Europe/Prague
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=104325544,total=350748672
   [junit4]   2> NOTE: All tests run in this JVM: [CurrencyFieldOpenExchangeTest, TestInfoStreamLogging, DirectUpdateHandlerOptimizeTest, NotRequiredUniqueKeyTest, HighlighterMaxOffsetTest, TestReplicationHandler, SolrRequestParserTest, CacheHeaderTest, TestSuggestSpellingConverter, TestDistributedSearch, MoreLikeThisHandlerTest, SpatialFilterTest, SynonymTokenizerTest, BJQParserTest, TestClusterStateMutator, TestPerFieldSimilarity, TestRTGBase, RecoveryAfterSoftCommitTest, HdfsBasicDistributedZkTest, SolrIndexSplitterTest, TestAnalyzedSuggestions, RAMDirectoryFactoryTest, TestDynamicFieldResource, TestReload, StatsComponentTest, TestCloudSchemaless, TestQuerySenderNoQuery, TestLuceneMatchVersion, SolrXmlInZkTest, DistributedQueryComponentOptimizationTest, DistanceUnitsTest, TestReqParamsAPI, ActionThrottleTest, HardAutoCommitTest, CursorPagingTest, TestSchemaVersionResource, DistributedFacetPivotWhiteBoxTest, SliceStateTest, URLClassifyProcessorTest, QueryEqualityTest, DistributedFacetPivotLargeTest, DirectUpdateHandlerTest, TestExtendedDismaxParser, TestSchemaSimilarityResource, SuggestComponentTest, PeerSyncTest, MergeStrategyTest, ConnectionManagerTest, TestBM25SimilarityFactory, AddSchemaFieldsUpdateProcessorFactoryTest, SyncSliceTest, TestRandomDVFaceting, TestExceedMaxTermLength, TestReversedWildcardFilterFactory, TestRemoteStreaming, AnalysisErrorHandlingTest, HdfsBasicDistributedZk2Test, HdfsRecoveryZkTest, HdfsSyncSliceTest, HdfsUnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, CachingDirectoryFactoryTest, TestInitParams, TestShardHandlerFactory, TestSolrConfigHandler, TestReplicationHandlerBackup, TestSolrConfigHandlerCloud, TestManagedResource, TestRestManager, TestFieldResource, TestManagedSchemaDynamicFieldResource, TestCloudManagedSchema, TestCollationFieldDocValues, TestSchemaManager, AnalyticsMergeStrategyTest, AnalyticsQueryTest, CursorMarkTest, RankQueryTest, TestCollapseQParserPlugin, TestComplexPhraseQParserPlugin, TestCustomSort, TestFieldSortValues, TestInitQParser, TestNoOpRegenerator, TestReRankQParserPlugin, TestRecoveryHdfs, TestSearcherReuse]
   [junit4] Completed on J2 in 0.68s, 1 test, 1 failure <<< FAILURES!

[...truncated 99 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:512: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 465 suites, 1857 tests, 5 errors, 2 failures, 56 ignored (9 assumptions)

Total time: 177 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 694885269 bytes
Compression is 0.0%
Took 2 min 11 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 757 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/757/

7 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=10802, name=Thread-4774, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=10802, name=Thread-4774, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40935/collection1
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:379)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40935/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:377)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 4 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1423420565492,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_LuceneVarGapDocFreqInterval_0.doc, _0_LuceneVarGapDocFreqInterval_0.tib, _0_LuceneVarGapDocFreqInterval_0.tiv, _1.fdt, _1.fdx, _1.fnm, _1.nvd, _1.nvm, _1.si, _1_LuceneVarGapDocFreqInterval_0.doc, _1_LuceneVarGapDocFreqInterval_0.tib, _1_LuceneVarGapDocFreqInterval_0.tiv, _2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]> but was:<[{indexVersion=1423420565492,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_LuceneVarGapDocFreqInterval_0.doc, _0_LuceneVarGapDocFreqInterval_0.tib, _0_LuceneVarGapDocFreqInterval_0.tiv, _1.fdt, _1.fdx, _1.fnm, _1.nvd, _1.nvm, _1.si, _1_LuceneVarGapDocFreqInterval_0.doc, _1_LuceneVarGapDocFreqInterval_0.tib, _1_LuceneVarGapDocFreqInterval_0.tiv, _2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}, {indexVersion=1423420565492,generation=3,filelist=[_2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, _6.cfe, _6.cfs, _6.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1423420565492,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_LuceneVarGapDocFreqInterval_0.doc, _0_LuceneVarGapDocFreqInterval_0.tib, _0_LuceneVarGapDocFreqInterval_0.tiv, _1.fdt, _1.fdx, _1.fnm, _1.nvd, _1.nvm, _1.si, _1_LuceneVarGapDocFreqInterval_0.doc, _1_LuceneVarGapDocFreqInterval_0.tib, _1_LuceneVarGapDocFreqInterval_0.tiv, _2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]> but was:<[{indexVersion=1423420565492,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_LuceneVarGapDocFreqInterval_0.doc, _0_LuceneVarGapDocFreqInterval_0.tib, _0_LuceneVarGapDocFreqInterval_0.tiv, _1.fdt, _1.fdx, _1.fnm, _1.nvd, _1.nvm, _1.si, _1_LuceneVarGapDocFreqInterval_0.doc, _1_LuceneVarGapDocFreqInterval_0.tib, _1_LuceneVarGapDocFreqInterval_0.tiv, _2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}, {indexVersion=1423420565492,generation=3,filelist=[_2.fdt, _2.fdx, _2.fnm, _2.nvd, _2.nvm, _2.si, _2_LuceneVarGapDocFreqInterval_0.doc, _2_LuceneVarGapDocFreqInterval_0.tib, _2_LuceneVarGapDocFreqInterval_0.tiv, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_LuceneVarGapDocFreqInterval_0.doc, _3_LuceneVarGapDocFreqInterval_0.tib, _3_LuceneVarGapDocFreqInterval_0.tiv, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_LuceneVarGapDocFreqInterval_0.doc, _4_LuceneVarGapDocFreqInterval_0.tib, _4_LuceneVarGapDocFreqInterval_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_LuceneVarGapDocFreqInterval_0.doc, _5_LuceneVarGapDocFreqInterval_0.tib, _5_LuceneVarGapDocFreqInterval_0.tiv, _6.cfe, _6.cfs, _6.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:8CBF87DD0F250C6A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32902/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32902/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:213CA337D1916F91]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32902/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38974/moz/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38974/moz/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:213CA337D1916F91]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38974/moz/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:19546/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19546/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:213CA337D1916F91]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33581/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33581/collection1
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:213CA337D1916F91]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33581/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33669/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33669/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A9689CED7F6D0269:213CA337D1916F91]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33669/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 10177 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/init-core-data-001
   [junit4]   2> 674560 T5871 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 674567 T5871 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 674568 T5872 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 674667 T5871 oasc.ZkTestServer.run start zk server on port:19497
   [junit4]   2> 674668 T5871 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674669 T5871 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674674 T5879 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@670831d1 name:ZooKeeperConnection Watcher:127.0.0.1:19497 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674674 T5871 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674675 T5871 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674675 T5871 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 674678 T5871 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674679 T5871 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674681 T5882 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e9b686 name:ZooKeeperConnection Watcher:127.0.0.1:19497/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674681 T5871 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674681 T5871 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674682 T5871 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 674684 T5871 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 674686 T5871 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 674688 T5871 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 674691 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 674691 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 674694 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 674695 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 674798 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 674799 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 674801 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 674801 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 674804 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 674804 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 674806 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 674807 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 674809 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 674810 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 674812 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 674812 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 674814 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 674815 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 674817 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 674818 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 674820 T5871 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 674820 T5871 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 676384 T5871 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores/collection1
   [junit4]   2> 676389 T5871 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 676394 T5871 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19504
   [junit4]   2> 676395 T5871 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 676396 T5871 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 676396 T5871 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001
   [junit4]   2> 676396 T5871 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/'
   [junit4]   2> 676444 T5871 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/solr.xml
   [junit4]   2> 676499 T5871 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores
   [junit4]   2> 676499 T5871 oasc.CoreContainer.<init> New CoreContainer 629815690
   [junit4]   2> 676500 T5871 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/]
   [junit4]   2> 676501 T5871 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 676502 T5871 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 676502 T5871 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 676502 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 676503 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 676503 T5871 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 676503 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 676504 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 676504 T5871 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 676504 T5871 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 676505 T5871 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 676505 T5871 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 676506 T5871 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 676506 T5871 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 676506 T5871 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 676507 T5871 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19497/solr
   [junit4]   2> 676507 T5871 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 676508 T5871 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 676509 T5871 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676511 T5894 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@467da479 name:ZooKeeperConnection Watcher:127.0.0.1:19497 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676511 T5871 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676512 T5871 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 676514 T5871 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676516 T5897 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38e00a42 name:ZooKeeperConnection Watcher:127.0.0.1:19497/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676516 T5871 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676518 T5871 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 676520 T5871 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 676523 T5871 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 676526 T5871 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 676528 T5871 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 676531 T5871 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 676532 T5871 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19503_
   [junit4]   2> 676533 T5871 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19503_
   [junit4]   2> 676535 T5871 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 676537 T5871 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 676539 T5871 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 676540 T5871 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:19503_
   [junit4]   2> 676541 T5871 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 676543 T5871 oasc.Overseer.start Overseer (id=93285139266142211-127.0.0.1:19503_-n_0000000000) starting
   [junit4]   2> 676545 T5871 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 676552 T5871 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 676553 T5899 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 676554 T5871 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 676556 T5871 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 676558 T5871 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 676561 T5898 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 676562 T5871 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores
   [junit4]   2> 676564 T5871 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores/collection1/
   [junit4]   2> 676565 T5871 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 676566 T5901 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 676567 T5901 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 676568 T5897 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 676568 T5901 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 676571 T5898 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19503_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19503"} current state version: 0
   [junit4]   2> 676571 T5898 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19503_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19503"}
   [junit4]   2> 676572 T5898 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 676572 T5898 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 676574 T5897 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 677568 T5901 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 677569 T5901 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 677570 T5901 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 677570 T5901 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 677571 T5901 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 677571 T5901 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores/collection1/'
   [junit4]   2> 677592 T5901 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 677596 T5901 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 677613 T5901 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 677647 T5901 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 677707 T5901 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 677808 T5901 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 677824 T5901 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 678268 T5901 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 678279 T5901 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 678282 T5901 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 678296 T5901 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 678302 T5901 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 678308 T5901 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 678309 T5901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 678309 T5901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 678310 T5901 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 678310 T5901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 678311 T5901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 678311 T5901 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 678311 T5901 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 678312 T5901 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/tempDir-001/control/data/
   [junit4]   2> 678312 T5901 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@49dc008c
   [junit4]   2> 678313 T5901 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/tempDir-001/control/data
   [junit4]   2> 678313 T5901 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/tempDir-001/control/data/index/
   [junit4]   2> 678313 T5901 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 678314 T5901 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/tempDir-001/control/data/index
   [junit4]   2> 678314 T5901 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=54.2900390625, floorSegmentMB=2.1103515625, forceMergeDeletesPctAllowed=1.74374449244591, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39245622290704707
   [junit4]   2> 678315 T5901 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@496bdb4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@480ac1e3),segFN=segments_1,generation=1}
   [junit4]   2> 678315 T5901 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 678319 T5901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 678319 T5901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 678320 T5901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 678320 T5901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 678320 T5901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 678320 T5901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 678321 T5901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 678321 T5901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 678322 T5901 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 678322 T5901 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 678323 T5901 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 678323 T5901 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 678324 T5901 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 678324 T5901 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 678324 T5901 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 678325 T5901 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 678325 T5901 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 678326 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 678327 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 678327 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 678328 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 678328 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 678329 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 678329 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 678330 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 678330 T5901 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 678331 T5901 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 678350 T5901 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 678352 T5901 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 678354 T5901 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 678356 T5901 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 678358 T5901 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 678362 T5901 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 678362 T5901 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 678363 T5901 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7ac1d39b
   [junit4]   2> 678364 T5901 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@496bdb4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@480ac1e3),segFN=segments_1,generation=1}
   [junit4]   2> 678364 T5901 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 678364 T5901 oass.SolrIndexSearcher.<init> Opening Searcher@2c9c7110[collection1] main
   [junit4]   2> 678364 T5901 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 678365 T5901 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 678366 T5901 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 678366 T5901 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 678366 T5901 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 678367 T5901 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 678367 T5901 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 678367 T5901 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 678368 T5901 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 678368 T5901 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 678369 T5902 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2c9c7110[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 678369 T5901 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 678370 T5905 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19503 collection:control_collection shard:shard1
   [junit4]   2> 678370 T5871 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 678371 T5871 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 678371 T5905 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 678375 T5871 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 678376 T5871 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678376 T5905 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 678377 T5909 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@212965f9 name:ZooKeeperConnection Watcher:127.0.0.1:19497/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678378 T5871 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678378 T5897 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678378 T5871 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 678379 T5905 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 678379 T5905 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 678380 T5898 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 678380 T5871 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C671 name=collection1 org.apache.solr.core.SolrCore@f7e88cf url=http://127.0.0.1:19503/collection1 node=127.0.0.1:19503_ C671_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:19503_, state=down, base_url=http://127.0.0.1:19503}
   [junit4]   2> 678380 T5905 C671 P19503 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19503/collection1/
   [junit4]   2> 678381 T5905 C671 P19503 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 678381 T5905 C671 P19503 oasc.SyncStrategy.syncToMe http://127.0.0.1:19503/collection1/ has no replicas
   [junit4]   2> 678381 T5905 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19503/collection1/ shard1
   [junit4]   2> 678381 T5905 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 678381 T5909 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 678381 T5897 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 678384 T5871 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 678387 T5910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678387 T5898 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19503",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 678491 T5909 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 678539 T5905 oasc.ZkController.register We are http://127.0.0.1:19503/collection1/ and leader is http://127.0.0.1:19503/collection1/
   [junit4]   2> 678540 T5905 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:19503
   [junit4]   2> 678540 T5905 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 678540 T5905 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 678540 T5905 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 678542 T5897 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678544 T5898 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19503_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19503"} current state version: 3
   [junit4]   2> 678545 T5898 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19503_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19503"}
   [junit4]   2> 678648 T5897 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 678648 T5909 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 679639 T5871 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001/cores/collection1
   [junit4]   2> 679642 T5871 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 679643 T5871 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 679645 T5871 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19512
   [junit4]   2> 679645 T5871 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 679646 T5871 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 679646 T5871 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001
   [junit4]   2> 679646 T5871 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001/'
   [junit4]   2> 679681 T5871 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001/solr.xml
   [junit4]   2> 679723 T5871 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001/cores
   [junit4]   2> 679724 T5871 oasc.CoreContainer.<init> New CoreContainer 2006562172
   [junit4]   2> 679724 T5871 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A9689CED7F6D0269-001/shard-1-001/]
   [junit4]   2> 679725 T5871 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 679725 T5871 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 679725 T5871 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 679725 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 679726 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 679726 T5871 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 679726 T5871 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
  

[...truncated too long message...]

n&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   2> 3299317 T10251 C1494 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&qt=/replication&command=indexversion&version=2&wt=javabin} status=0 QTime=0 
   [junit4]   1> [index]
   [junit4]   1> [replication.properties, index.properties, index.20150208134126959]
   [junit4]   2> 3299319 T9642 oas.SolrTestCaseJ4.tearDown ###Ending doTestStressReplication
   [junit4]   2> 3299319 T9642 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1082626756
   [junit4]   2> 3299320 T9642 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6a5a8733
   [junit4]   2> 3299321 T9642 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=80,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=39490,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 3299321 T9642 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 3299321 T9642 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 3299322 T9642 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 3299328 T9642 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 3299328 T9642 oasc.CachingDirectoryFactory.close Closing NRTCachingDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 3299329 T9642 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 3299329 T9642 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data/index
   [junit4]   2> 3299330 T9642 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 3299330 T9642 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-001/collection1/data
   [junit4]   2> 3299331 T9642 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3299523 T9642 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=949580629
   [junit4]   2> 3299524 T9642 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@8974f52
   [junit4]   2> 3299524 T9642 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 3299525 T9642 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 3299525 T9642 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 3299525 T9642 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 3299532 T9642 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 3299534 T9642 oasc.CachingDirectoryFactory.close Closing NRTCachingDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 3299534 T9642 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data/index.20150208134126959 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data/index.20150208134126959;done=false>>]
   [junit4]   2> 3299534 T9642 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data/index.20150208134126959
   [junit4]   2> 3299535 T9642 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 3299535 T9642 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001/solr-instance-002/collection1/data
   [junit4]   2> 3299536 T9642 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3299793 T9642 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler A9689CED7F6D0269-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {name=PostingsFormat(name=LuceneVarGapDocFreqInterval), newname=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=sq, timezone=America/Kentucky/Louisville
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=82824040,total=283639808
   [junit4]   2> NOTE: All tests run in this JVM: [TestRealTimeGet, DistribDocExpirationUpdateProcessorTest, FileBasedSpellCheckerTest, ConnectionManagerTest, RequestLoggingTest, LukeRequestHandlerTest, QueryEqualityTest, SliceStateTest, MoreLikeThisHandlerTest, TestDynamicLoading, SuggestComponentTest, AddBlockUpdateTest, ChaosMonkeyNothingIsSafeTest, CurrencyFieldOpenExchangeTest, HdfsCollectionsAPIDistributedZkTest, TestRecoveryHdfs, DistributedDebugComponentTest, TestHighlightDedupGrouping, CursorPagingTest, TestInitParams, TestComponentsName, TestSolrQueryParser, TestBM25SimilarityFactory, TestHashPartitioner, RollingRestartTest, TestObjectReleaseTracker, TestDistributedMissingSort, TestPHPSerializedResponseWriter, TestDistributedSearch, RAMDirectoryFactoryTest, TestDefaultSearchFieldResource, TestReload, TestCSVResponseWriter, TestSolrQueryParserDefaultOperatorResource, ExternalCollectionsTest, TestReqParamsAPI, DistributedQueryComponentOptimizationTest, SynonymTokenizerTest, TestInfoStreamLogging, LeaderElectionIntegrationTest, TestLuceneMatchVersion, TestIntervalFaceting, DirectUpdateHandlerTest, TestReplicationHandler]
   [junit4] Completed on J2 in 802.69s, 13 tests, 1 failure <<< FAILURES!

[...truncated 51 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:512: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 464 suites, 1856 tests, 6 errors, 1 failure, 56 ignored (9 assumptions)

Total time: 174 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 751529163 bytes
Compression is 0.0%
Took 2 min 9 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 756 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/756/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.CloudExitableDirectoryReaderTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:10986/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:10986/collection1]
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:349)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1009)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1384)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.assertPartialResults(CloudExitableDirectoryReaderTest.java:102)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.doTimeoutTests(CloudExitableDirectoryReaderTest.java:86)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.test(CloudExitableDirectoryReaderTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:10986/collection1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	... 51 more


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1423324250088,generation=2,filelist=[_2.cfe, _2.cfs, _2.si, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_MockRandom_0.doc, _3_MockRandom_0.sd, _3_MockRandom_0.tmp, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_MockRandom_0.doc, _4_MockRandom_0.sd, _4_MockRandom_0.tib, _4_MockRandom_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_MockRandom_0.doc, _5_MockRandom_0.sd, _5_MockRandom_0.tmp, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_MockRandom_0.doc, _6_MockRandom_0.sd, _6_MockRandom_0.tim, _6_MockRandom_0.tip, segments_2]}]> but was:<[{indexVersion=1423324250088,generation=3,filelist=[_2.cfe, _2.cfs, _2.si, _7.cfe, _7.cfs, _7.si, segments_3]}, {indexVersion=1423324250088,generation=2,filelist=[_2.cfe, _2.cfs, _2.si, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_MockRandom_0.doc, _3_MockRandom_0.sd, _3_MockRandom_0.tmp, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_MockRandom_0.doc, _4_MockRandom_0.sd, _4_MockRandom_0.tib, _4_MockRandom_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_MockRandom_0.doc, _5_MockRandom_0.sd, _5_MockRandom_0.tmp, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_MockRandom_0.doc, _6_MockRandom_0.sd, _6_MockRandom_0.tim, _6_MockRandom_0.tip, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1423324250088,generation=2,filelist=[_2.cfe, _2.cfs, _2.si, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_MockRandom_0.doc, _3_MockRandom_0.sd, _3_MockRandom_0.tmp, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_MockRandom_0.doc, _4_MockRandom_0.sd, _4_MockRandom_0.tib, _4_MockRandom_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_MockRandom_0.doc, _5_MockRandom_0.sd, _5_MockRandom_0.tmp, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_MockRandom_0.doc, _6_MockRandom_0.sd, _6_MockRandom_0.tim, _6_MockRandom_0.tip, segments_2]}]> but was:<[{indexVersion=1423324250088,generation=3,filelist=[_2.cfe, _2.cfs, _2.si, _7.cfe, _7.cfs, _7.si, segments_3]}, {indexVersion=1423324250088,generation=2,filelist=[_2.cfe, _2.cfs, _2.si, _3.fdt, _3.fdx, _3.fnm, _3.nvd, _3.nvm, _3.si, _3_MockRandom_0.doc, _3_MockRandom_0.sd, _3_MockRandom_0.tmp, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_MockRandom_0.doc, _4_MockRandom_0.sd, _4_MockRandom_0.tib, _4_MockRandom_0.tiv, _5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_MockRandom_0.doc, _5_MockRandom_0.sd, _5_MockRandom_0.tmp, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_MockRandom_0.doc, _6_MockRandom_0.sd, _6_MockRandom_0.tim, _6_MockRandom_0.tip, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:3E56B87FF82A6851]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62831/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62831/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B81A34F88626652:93D59C95269E0BAA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10357 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/init-core-data-001
   [junit4]   2> 320543 T4784 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /t/p
   [junit4]   2> 320547 T4784 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 320548 T4785 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 320648 T4784 oasc.ZkTestServer.run start zk server on port:60989
   [junit4]   2> 320649 T4784 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 320650 T4784 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 320653 T4792 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34115ba2 name:ZooKeeperConnection Watcher:127.0.0.1:60989 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 320654 T4784 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 320654 T4784 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 320654 T4784 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 320657 T4784 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 320657 T4784 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 320658 T4795 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@eb2dd58 name:ZooKeeperConnection Watcher:127.0.0.1:60989/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 320658 T4784 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 320659 T4784 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 320659 T4784 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 320661 T4784 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 320662 T4784 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 320663 T4784 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 320664 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 320665 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 320666 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 320667 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 320668 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 320669 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 320670 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 320670 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 320672 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 320672 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 320673 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 320674 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 320675 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 320675 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 320677 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 320677 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 320678 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 320679 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 320680 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 320680 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 320681 T4784 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 320682 T4784 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 321876 T4784 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores/collection1
   [junit4]   2> 321879 T4784 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 321883 T4784 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60994
   [junit4]   2> 321883 T4784 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 321884 T4784 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 321884 T4784 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001
   [junit4]   2> 321884 T4784 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/'
   [junit4]   2> 321917 T4784 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/solr.xml
   [junit4]   2> 321957 T4784 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores
   [junit4]   2> 321957 T4784 oasc.CoreContainer.<init> New CoreContainer 1906687905
   [junit4]   2> 321957 T4784 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/]
   [junit4]   2> 321958 T4784 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 321959 T4784 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 321959 T4784 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 321959 T4784 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 321959 T4784 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 321959 T4784 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 321960 T4784 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 321960 T4784 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 321960 T4784 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 321960 T4784 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 321961 T4784 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 321961 T4784 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 321961 T4784 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 321962 T4784 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 321962 T4784 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 321962 T4784 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60989/solr
   [junit4]   2> 321962 T4784 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 321963 T4784 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 321963 T4784 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 321965 T4807 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@589ce2ec name:ZooKeeperConnection Watcher:127.0.0.1:60989 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 321965 T4784 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 321966 T4784 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 321968 T4784 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 321969 T4810 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19079f96 name:ZooKeeperConnection Watcher:127.0.0.1:60989/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 321969 T4784 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 321970 T4784 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 321972 T4784 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 321973 T4784 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 321975 T4784 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 321976 T4784 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 321978 T4784 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 321978 T4784 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60993_t%2Fp
   [junit4]   2> 321979 T4784 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60993_t%2Fp
   [junit4]   2> 321980 T4784 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 321981 T4784 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 321982 T4784 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 321983 T4784 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60993_t%2Fp
   [junit4]   2> 321984 T4784 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 321985 T4784 oasc.Overseer.start Overseer (id=93278880620019715-127.0.0.1:60993_t%2Fp-n_0000000000) starting
   [junit4]   2> 321986 T4784 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 321990 T4784 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 321991 T4812 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 321991 T4784 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 321993 T4784 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 321993 T4784 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 321995 T4811 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 321996 T4784 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores
   [junit4]   2> 321998 T4784 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores/collection1/
   [junit4]   2> 321998 T4784 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 321998 T4814 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 321999 T4814 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 321999 T4814 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 321999 T4810 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 322001 T4811 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60993/t/p",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:60993_t%2Fp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 322001 T4811 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60993/t/p",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:60993_t%2Fp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 322002 T4811 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 322002 T4811 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 322003 T4810 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 323000 T4814 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 323001 T4814 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 323002 T4814 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 323002 T4814 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 323003 T4814 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 323003 T4814 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores/collection1/'
   [junit4]   2> 323020 T4814 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 323023 T4814 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 323034 T4814 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 323058 T4814 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 323110 T4814 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 323112 T4814 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 323124 T4814 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 323542 T4814 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 323554 T4814 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 323557 T4814 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 323569 T4814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 323574 T4814 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 323579 T4814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 323579 T4814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 323580 T4814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 323580 T4814 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 323581 T4814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 323581 T4814 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 323582 T4814 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 323582 T4814 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 323582 T4814 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/tempDir-001/control/data/
   [junit4]   2> 323583 T4814 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4a62a8af
   [junit4]   2> 323584 T4814 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/tempDir-001/control/data
   [junit4]   2> 323584 T4814 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/tempDir-001/control/data/index/
   [junit4]   2> 323584 T4814 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 323585 T4814 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 1B81A34F88626652-001/tempDir-001/control/data/index
   [junit4]   2> 323585 T4814 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2104fcda
   [junit4]   2> 323586 T4814 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e745d25 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@398f5060),segFN=segments_1,generation=1}
   [junit4]   2> 323586 T4814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 323592 T4814 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 323593 T4814 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 323593 T4814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 323594 T4814 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 323594 T4814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 323594 T4814 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 323595 T4814 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 323595 T4814 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 323595 T4814 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 323596 T4814 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 323596 T4814 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 323597 T4814 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 323597 T4814 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 323598 T4814 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 323599 T4814 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 323599 T4814 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 323600 T4814 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 323600 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 323602 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 323602 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 323603 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 323604 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 323604 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 323605 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 323605 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 323606 T4814 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 323606 T4814 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 323632 T4814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 323634 T4814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 323636 T4814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 323638 T4814 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 323640 T4814 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 323643 T4814 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 323644 T4814 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 323644 T4814 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=6, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 323645 T4814 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e745d25 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@398f5060),segFN=segments_1,generation=1}
   [junit4]   2> 323645 T4814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 323646 T4814 oass.SolrIndexSearcher.<init> Opening Searcher@12813c5d[collection1] main
   [junit4]   2> 323646 T4814 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 323647 T4814 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 323647 T4814 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 323648 T4814 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 323648 T4814 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 323648 T4814 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 323649 T4814 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 323649 T4814 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 323649 T4814 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 323650 T4814 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 323651 T4815 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@12813c5d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 323651 T4814 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 323652 T4818 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60993/t/p collection:control_collection shard:shard1
   [junit4]   2> 323652 T4784 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 323653 T4784 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 323653 T4818 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 323656 T4784 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 323656 T4784 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 323658 T4818 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 323658 T4822 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b4cfc4a name:ZooKeeperConnection Watcher:127.0.0.1:60989/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 323658 T4784 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 323659 T4784 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 323660 T4810 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 323660 T4818 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 323660 T4818 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 323660 T4784 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 323660 T4811 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C381 name=collection1 org.apache.solr.core.SolrCore@16a0d53 url=http://127.0.0.1:60993/t/p/collection1 node=127.0.0.1:60993_t%2Fp C381_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:60993/t/p, state=down, node_name=127.0.0.1:60993_t%2Fp, core=collection1}
   [junit4]   2> 323660 T4818 C381 P60993 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60993/t/p/collection1/
   [junit4]   2> 323661 T4818 C381 P60993 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 323661 T4818 C381 P60993 oasc.SyncStrategy.syncToMe http://127.0.0.1:60993/t/p/collection1/ has no replicas
   [junit4]   2> 323661 T4818 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60993/t/p/collection1/ shard1
   [junit4]   2> 323662 T4818 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 323662 T4822 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 323662 T4810 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 323663 T4784 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 323666 T4823 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 323666 T4811 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60993/t/p",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 323769 T4822 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 323818 T4818 oasc.ZkController.register We are http://127.0.0.1:60993/t/p/collection1/ and leader is http://127.0.0.1:60993/t/p/collection1/
   [junit4]   2> 323819 T4818 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60993/t/p
   [junit4]   2> 323819 T4818 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 323819 T4818 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 323819 T4818 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 323820 T4810 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 323822 T4811 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60993/t/p",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:60993_t%2Fp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 323822 T4811 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60993/t/p",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:60993_t%2Fp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 323925 T4822 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating..

[...truncated too long message...]

###Ending doTestDetails
   [junit4]   2> 2281997 T10075 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1950954557
   [junit4]   2> 2281998 T10075 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6d6f3fb4
   [junit4]   2> 2281998 T10075 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 2281999 T10075 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2281999 T10075 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2281999 T10075 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2282001 T10075 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2282052 T10075 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2282052 T10075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 2282052 T10075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data
   [junit4]   2> 2282053 T10075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 2282053 T10075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-001/collection1/data/index
   [junit4]   2> 2282053 T10075 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2282200 T10075 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1131916603
   [junit4]   2> 2282201 T10075 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2fb2161c
   [junit4]   2> 2296921 T10939 oash.SnapPuller.fetchLatestIndex ERROR Master at: http://127.0.0.1:42180/solr/collection1 is not available. Index fetch failed. Exception: IOException occured when talking to server at: http://127.0.0.1:42180/solr/collection1
   [junit4]   2> 2296922 T10075 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 2296922 T10075 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2296923 T10075 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2296923 T10075 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2296925 T10075 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2296925 T10075 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2296926 T10075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data/index;done=false>>]
   [junit4]   2> 2296926 T10075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data/index
   [junit4]   2> 2296926 T10075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 2296927 T10075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001/solr-instance-002/collection1/data
   [junit4]   2> 2296928 T10075 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2297140 T10075 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 1B81A34F88626652-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {newname=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom), name=PostingsFormat(name=MockRandom)}, docValues:{}, sim=DefaultSimilarity, locale=pt, timezone=America/Monterrey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=234885904,total=372244480
   [junit4]   2> NOTE: All tests run in this JVM: [OpenExchangeRatesOrgProviderTest, XmlUpdateRequestHandlerTest, TestUpdate, TestBulkSchemaConcurrent, TestManagedSchemaFieldTypeResource, DistributedIntervalFacetingTest, TestCollationField, DistributedSpellCheckComponentTest, RecoveryZkTest, TestTolerantSearch, TestNRTOpen, HdfsChaosMonkeySafeLeaderTest, BlockDirectoryTest, ParsingFieldUpdateProcessorsTest, TestRequestStatusCollectionAPI, TestAtomicUpdateErrorCases, TestFieldCollectionResource, ReplicationFactorTest, DocExpirationUpdateProcessorFactoryTest, TestSolrDeletionPolicy2, DocValuesMissingTest, ShardRoutingCustomTest, OverseerCollectionProcessorTest, TimeZoneUtilsTest, DirectSolrSpellCheckerTest, SolrIndexConfigTest, DateRangeFieldTest, MinimalSchemaTest, CursorMarkTest, SolrCmdDistributorTest, SSLMigrationTest, TestCloudPivotFacet, DistributedQueueTest, TestHighFrequencyDictionaryFactory, DOMUtilTest, MBeansHandlerTest, SimpleFacetsTest, TestRebalanceLeaders, TestSolrIndexConfig, RemoteQueryErrorTest, StatelessScriptUpdateProcessorFactoryTest, TestCoreContainer, OverseerTest, QueryParsingTest, TestPartialUpdateDeduplication, TestTrie, ZkSolrClientTest, SearchHandlerTest, LeaderInitiatedRecoveryOnCommitTest, TestIBSimilarityFactory, DebugComponentTest, TestDynamicFieldCollectionResource, SuggesterTest, TestSearcherReuse, TestMissingGroups, TestImplicitCoreProperties, FastVectorHighlighterTest, ChaosMonkeySafeLeaderTest, TestReplicationHandler]
   [junit4] Completed on J0 in 975.84s, 13 tests, 1 failure <<< FAILURES!

[...truncated 311 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 464 suites, 1856 tests, 6 errors, 2 failures, 56 ignored (9 assumptions)

Total time: 146 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 702312103 bytes
Compression is 0.0%
Took 2 min 25 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 755 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/755/

7 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
The Monkey ran for over 20 seconds and no jetties were stopped - this is worth investigating!

Stack Trace:
java.lang.AssertionError: The Monkey ran for over 20 seconds and no jetties were stopped - this is worth investigating!
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7:3280F91A0B6CF80F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.ChaosMonkey.stopTheMonkey(ChaosMonkey.java:537)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:183)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
Some resources were not closed, shutdown, or released.

Stack Trace:
java.lang.AssertionError: Some resources were not closed, shutdown, or released.
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:213)
	at sun.reflect.GeneratedMethodAccessor34.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=7040, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)    2) Thread[id=7135, name=StopableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    3) Thread[id=7169, name=zkCallback-857-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=7168, name=zkCallback-857-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=7136, name=zkCallback-857-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=7042, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)    7) Thread[id=7043, name=StopableSearchThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.AbstractFullDistribZkTestBase$StopableSearchThread.run(AbstractFullDistribZkTestBase.java:1425)    8) Thread[id=7041, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)    9) Thread[id=7134, name=StopableSearchThread-SendThread(127.0.0.1:19069), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=7040, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)
   2) Thread[id=7135, name=StopableSearchThread-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   3) Thread[id=7169, name=zkCallback-857-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=7168, name=zkCallback-857-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=7136, name=zkCallback-857-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=7042, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)
   7) Thread[id=7043, name=StopableSearchThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase$StopableSearchThread.run(AbstractFullDistribZkTestBase.java:1425)
   8) Thread[id=7041, name=StopableIndexingThread, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:121)
   9) Thread[id=7134, name=StopableSearchThread-SendThread(127.0.0.1:19069), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7043, name=StopableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.io.PrintStream.write(PrintStream.java:478)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)         at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)         at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)         at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)         at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)         at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)         at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)         at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)         at org.apache.log4j.Category.callAppenders(Category.java:206)         at org.apache.log4j.Category.forcedLog(Category.java:391)         at org.apache.log4j.Category.log(Category.java:856)         at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:808)         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)         at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)         at org.apache.solr.cloud.AbstractFullDistribZkTestBase$StopableSearchThread.run(AbstractFullDistribZkTestBase.java:1414)    2) Thread[id=7134, name=StopableSearchThread-SendThread(127.0.0.1:19069), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:205)         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:243)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:102)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)    3) Thread[id=7041, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.io.PrintStream.println(PrintStream.java:806)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)    4) Thread[id=7136, name=zkCallback-857-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=7042, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.io.PrintStream.println(PrintStream.java:806)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)    6) Thread[id=7169, name=zkCallback-857-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=7040, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Class.getGenericSignature(Native Method)         at java.lang.Class.getGenericSuperclass(Class.java:690)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.getBoundFields(ReflectiveTypeAdapterFactory.java:120)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.create(ReflectiveTypeAdapterFactory.java:65)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.MiniGson.getAdapter(MiniGson.java:92)         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.Gson.toJson(Gson.java:504)         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:87)         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$4.write(SlaveMain.java:410)         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)         at java.io.PrintStream.flush(PrintStream.java:338)         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)         at java.io.PrintStream.write(PrintStream.java:482)         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)         at java.io.PrintStream.write(PrintStream.java:527)         at java.io.PrintStream.print(PrintStream.java:669)         at java.io.PrintStream.println(PrintStream.java:823)         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)         at java.lang.Throwable.printStackTrace(Throwable.java:658)         at java.lang.Throwable.printStackTrace(Throwable.java:643)         at java.lang.Throwable.printStackTrace(Throwable.java:634)         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:108)    8) Thread[id=7168, name=zkCallback-857-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7043, name=StopableSearchThread, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.io.PrintStream.write(PrintStream.java:478)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
        at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
        at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
        at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
        at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
        at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
        at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
        at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
        at org.apache.log4j.Category.callAppenders(Category.java:206)
        at org.apache.log4j.Category.forcedLog(Category.java:391)
        at org.apache.log4j.Category.log(Category.java:856)
        at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:808)
        at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
        at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
        at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
        at org.apache.solr.cloud.AbstractFullDistribZkTestBase$StopableSearchThread.run(AbstractFullDistribZkTestBase.java:1414)
   2) Thread[id=7134, name=StopableSearchThread-SendThread(127.0.0.1:19069), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:205)
        at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:243)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:102)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   3) Thread[id=7041, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.io.PrintStream.println(PrintStream.java:806)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)
   4) Thread[id=7136, name=zkCallback-857-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=7042, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.io.PrintStream.println(PrintStream.java:806)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)
   6) Thread[id=7169, name=zkCallback-857-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=7040, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Class.getGenericSignature(Native Method)
        at java.lang.Class.getGenericSuperclass(Class.java:690)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.getBoundFields(ReflectiveTypeAdapterFactory.java:120)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.create(ReflectiveTypeAdapterFactory.java:65)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.MiniGson.getAdapter(MiniGson.java:92)
        at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.Gson.toJson(Gson.java:504)
        at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:87)
        at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain$4.write(SlaveMain.java:410)
        at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
        at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
        at java.io.PrintStream.flush(PrintStream.java:338)
        at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
        at java.io.PrintStream.write(PrintStream.java:482)
        at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
        at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
        at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
        at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
        at java.io.PrintStream.write(PrintStream.java:527)
        at java.io.PrintStream.print(PrintStream.java:669)
        at java.io.PrintStream.println(PrintStream.java:823)
        at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
        at java.lang.Throwable.printStackTrace(Throwable.java:658)
        at java.lang.Throwable.printStackTrace(Throwable.java:643)
        at java.lang.Throwable.printStackTrace(Throwable.java:634)
        at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:108)
   8) Thread[id=7168, name=zkCallback-857-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest:     1) Thread[id=884, name=OverseerThreadFactory-338-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1822)         at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1709)         at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:621)         at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2861)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest: 
   1) Thread[id=884, name=OverseerThreadFactory-338-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1822)
        at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1709)
        at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:621)
        at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2861)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7]:0)


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7:3280F91A0B6CF80F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7:3280F91A0B6CF80F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9826 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/init-core-data-001
   [junit4]   2> 101986 T319 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 102000 T319 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 102001 T320 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102101 T319 oasc.ZkTestServer.run start zk server on port:48537
   [junit4]   2> 102102 T319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102103 T319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102106 T327 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@660185c5 name:ZooKeeperConnection Watcher:127.0.0.1:48537 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102106 T319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102107 T319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102108 T319 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 102111 T319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102112 T319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102113 T330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ec2c455 name:ZooKeeperConnection Watcher:127.0.0.1:48537/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102113 T319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102114 T319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102114 T319 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 102117 T319 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 102120 T319 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 102122 T319 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 102125 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 102126 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 102130 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 102131 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 102134 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102135 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102137 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 102138 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 102141 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 102142 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 102144 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 102145 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 102148 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 102148 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 102151 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102152 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102155 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102156 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102159 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 102160 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 102163 T319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 102164 T319 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 103506 T319 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores/collection1
   [junit4]   2> 103510 T319 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 103515 T319 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48543
   [junit4]   2> 103515 T319 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 103516 T319 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 103516 T319 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001
   [junit4]   2> 103516 T319 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/'
   [junit4]   2> 103551 T319 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/solr.xml
   [junit4]   2> 103599 T319 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores
   [junit4]   2> 103599 T319 oasc.CoreContainer.<init> New CoreContainer 1313420669
   [junit4]   2> 103599 T319 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/]
   [junit4]   2> 103600 T319 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 103601 T319 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 103601 T319 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 103601 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 103601 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 103602 T319 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 103602 T319 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 103602 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 103602 T319 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 103603 T319 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 103603 T319 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 103603 T319 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 103604 T319 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 103604 T319 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 103604 T319 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 103605 T319 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48537/solr
   [junit4]   2> 103647 T319 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 103647 T319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103648 T319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103650 T342 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@147f7858 name:ZooKeeperConnection Watcher:127.0.0.1:48537 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103650 T319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103651 T319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103670 T319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103671 T345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72181eb5 name:ZooKeeperConnection Watcher:127.0.0.1:48537/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103672 T319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103687 T319 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 103700 T319 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 103707 T319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 103710 T319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 103713 T319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 103728 T319 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 103731 T319 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48542_
   [junit4]   2> 103738 T319 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48542_
   [junit4]   2> 103742 T319 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 103745 T319 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 103748 T319 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 103765 T319 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48542_
   [junit4]   2> 103766 T319 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 103768 T319 oasc.Overseer.start Overseer (id=93273212448079875-127.0.0.1:48542_-n_0000000000) starting
   [junit4]   2> 103786 T319 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 103821 T319 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 103838 T347 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 103838 T319 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 103841 T319 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 103843 T319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 103865 T346 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 103866 T319 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores
   [junit4]   2> 103884 T319 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores/collection1/
   [junit4]   2> 103884 T319 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 103885 T349 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 103885 T349 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 103898 T345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103898 T349 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 103909 T346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48542",
   [junit4]   2> 	  "node_name":"127.0.0.1:48542_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 103965 T346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48542",
   [junit4]   2> 	  "node_name":"127.0.0.1:48542_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 103973 T346 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 104019 T346 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 104034 T345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104899 T349 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 104899 T349 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 104900 T349 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 104901 T349 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 104902 T349 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 104903 T349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores/collection1/'
   [junit4]   2> 104961 T349 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 104991 T349 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 105032 T349 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 105063 T349 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 105128 T349 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 105130 T349 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 105145 T349 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 105568 T349 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 105581 T349 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 105585 T349 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 105611 T349 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105617 T349 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105622 T349 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105624 T349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105624 T349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105625 T349 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105626 T349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105626 T349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105627 T349 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 105627 T349 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 105628 T349 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/tempDir-001/control/data/
   [junit4]   2> 105628 T349 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@314cd8da
   [junit4]   2> 105644 T349 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/tempDir-001/control/data
   [junit4]   2> 105645 T349 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/tempDir-001/control/data/index/
   [junit4]   2> 105645 T349 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 105646 T349 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/tempDir-001/control/data/index
   [junit4]   2> 105646 T349 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=21, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 105647 T349 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@3473f5dd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29129b12),segFN=segments_1,generation=1}
   [junit4]   2> 105647 T349 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105657 T349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 105735 T349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 105736 T349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 105736 T349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105737 T349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105737 T349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 105738 T349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105738 T349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105738 T349 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 105748 T349 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 105749 T349 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 105750 T349 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 105751 T349 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 105752 T349 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 105753 T349 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 105754 T349 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 105756 T349 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 105757 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 105758 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 105760 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 105761 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 105762 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 105763 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 105764 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 105765 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 105767 T349 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 105768 T349 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 105790 T349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105792 T349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105794 T349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105796 T349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105798 T349 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 105802 T349 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 105802 T349 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 105804 T349 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=8, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 105804 T349 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@3473f5dd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29129b12),segFN=segments_1,generation=1}
   [junit4]   2> 105805 T349 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105805 T349 oass.SolrIndexSearcher.<init> Opening Searcher@6b4ee030[collection1] main
   [junit4]   2> 105806 T349 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 105807 T349 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 105808 T349 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 105820 T349 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 105820 T349 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 105820 T349 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105821 T349 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 105822 T349 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105822 T349 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 105822 T349 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 105825 T350 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b4ee030[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 105830 T349 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 105831 T353 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48542 collection:control_collection shard:shard1
   [junit4]   2> 105832 T319 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 105832 T319 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 105869 T353 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 105877 T353 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 105879 T345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105880 T353 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 105880 T353 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 105881 T346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 105886 T319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 105887 T319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C58 name=collection1 org.apache.solr.core.SolrCore@691eefce url=http://127.0.0.1:48542/collection1 node=127.0.0.1:48542_ C58_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:48542, node_name=127.0.0.1:48542_, state=down, core=collection1}
   [junit4]   2> 105888 T353 C58 P48542 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48542/collection1/
   [junit4]   2> 105889 T353 C58 P48542 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 105889 T357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b91bcf2 name:ZooKeeperConnection Watcher:127.0.0.1:48537/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 105889 T353 C58 P48542 oasc.SyncStrategy.syncToMe http://127.0.0.1:48542/collection1/ has no replicas
   [junit4]   2> 105890 T319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 105890 T353 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48542/collection1/ shard1
   [junit4]   2> 105891 T319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 105892 T319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 105893 T345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105902 T319 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 105903 T353 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 105911 T358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105913 T346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48542",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 106019 T357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 106065 T353 oasc.ZkController.register We are http://127.0.0.1:48542/collection1/ and leader is http://127.0.0.1:48542/collection1/
   [junit4]   2> 106065 T353 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48542
   [junit4]   2> 106065 T353 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 106066 T353 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 106066 T353 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 106068 T345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 106072 T346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48542",
   [junit4]   2> 	  "node_name":"127.0.0.1:48542_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 106073 T346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48542",
   [junit4]   2> 	  "node_name":"127.0.0.1:48542_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 106179 T357 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 106179 T345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107710 T319 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001/cores/collection1
   [junit4]   2> 107714 T319 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 107715 T319 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 107718 T319 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48560
   [junit4]   2> 107719 T319 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 107720 T319 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 107720 T319 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001
   [junit4]   2> 107720 T319 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001/'
   [junit4]   2> 107775 T319 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001/solr.xml
   [junit4]   2> 107843 T319 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001/cores
   [junit4]   2> 107844 T319 oasc.CoreContainer.<init> New CoreContainer 1908532237
   [junit4]   2> 107844 T319 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BAD4C6C0A59095F7-001/shard-1-001/]
   [junit4]   2> 107846 T319 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 107846 T319 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 107847 T319 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 107847 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 107847 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 107848 T319 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 107848 T319 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 107848 T319 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 107849 T319 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 107849 T319 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 107850 T319 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 107850 T319 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 107851 T319 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 107851 T319 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 107852 T319 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 107852 T319 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48537/solr
   [junit4]   2> 107853 T319 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 107853 T319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 107854 T319 oascc.ConnectionManager.waitForConnected Waiting for clie

[...truncated too long message...]

ategory.java:206)
   [junit4]    >         at org.apache.log4j.Category.forcedLog(Category.java:391)
   [junit4]    >         at org.apache.log4j.Category.log(Category.java:856)
   [junit4]    >         at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:540)
   [junit4]    >         at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:808)
   [junit4]    >         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    >         at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    >         at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
   [junit4]    >         at org.apache.solr.cloud.AbstractFullDistribZkTestBase$StopableSearchThread.run(AbstractFullDistribZkTestBase.java:1414)
   [junit4]    >    2) Thread[id=7134, name=StopableSearchThread-SendThread(127.0.0.1:19069), state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:205)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:243)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:102)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]    >    3) Thread[id=7041, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.io.PrintStream.println(PrintStream.java:806)
   [junit4]    >         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)
   [junit4]    >    4) Thread[id=7136, name=zkCallback-857-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=7042, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.io.PrintStream.println(PrintStream.java:806)
   [junit4]    >         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:107)
   [junit4]    >    6) Thread[id=7169, name=zkCallback-857-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=7040, name=StopableIndexingThread, state=BLOCKED, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Class.getGenericSignature(Native Method)
   [junit4]    >         at java.lang.Class.getGenericSuperclass(Class.java:690)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.getBoundFields(ReflectiveTypeAdapterFactory.java:120)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.ReflectiveTypeAdapterFactory.create(ReflectiveTypeAdapterFactory.java:65)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.internal.bind.MiniGson.getAdapter(MiniGson.java:92)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.dependencies.com.google.gson.Gson.toJson(Gson.java:504)
   [junit4]    >         at com.carrotsearch.ant.tasks.junit4.events.Serializer.serialize(Serializer.java:87)
   [junit4]    >         at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82)
   [junit4]    >         at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140)
   [junit4]    >         at java.io.PrintStream.flush(PrintStream.java:338)
   [junit4]    >         at java.io.FilterOutputStream.flush(FilterOutputStream.java:140)
   [junit4]    >         at java.io.PrintStream.write(PrintStream.java:482)
   [junit4]    >         at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
   [junit4]    >         at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
   [junit4]    >         at sun.nio.cs.StreamEncoder.flushBuffer(StreamEncoder.java:104)
   [junit4]    >         at java.io.OutputStreamWriter.flushBuffer(OutputStreamWriter.java:185)
   [junit4]    >         at java.io.PrintStream.write(PrintStream.java:527)
   [junit4]    >         at java.io.PrintStream.print(PrintStream.java:669)
   [junit4]    >         at java.io.PrintStream.println(PrintStream.java:823)
   [junit4]    >         at java.lang.Throwable$WrappedPrintStream.println(Throwable.java:748)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:658)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:643)
   [junit4]    >         at java.lang.Throwable.printStackTrace(Throwable.java:634)
   [junit4]    >         at org.apache.solr.cloud.StopableIndexingThread.run(StopableIndexingThread.java:108)
   [junit4]    >    8) Thread[id=7168, name=zkCallback-857-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:226)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:359)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:942)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BAD4C6C0A59095F7]:0)
   [junit4] Completed on J0 in 148.73s, 1 test, 2 failures, 2 errors <<< FAILURES!

[...truncated 359 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 464 suites, 1856 tests, 4 suite-level errors, 2 errors, 1 failure, 332 ignored (9 assumptions)

Total time: 133 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 666466107 bytes
Compression is 0.0%
Took 2 min 2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 754 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/754/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
expected:<0> but was:<1>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<1>
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32937/_t/sk/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32937/_t/sk/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9787 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/init-core-data-001
   [junit4]   2> 140677 T489 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h_ynf/
   [junit4]   2> 140682 T489 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 140683 T490 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 140783 T489 oasc.ZkTestServer.run start zk server on port:27370
   [junit4]   2> 140784 T489 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 140785 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140787 T497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a491af4 name:ZooKeeperConnection Watcher:127.0.0.1:27370 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140788 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140788 T489 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 140788 T489 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 140791 T489 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 140791 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140793 T500 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27892ea3 name:ZooKeeperConnection Watcher:127.0.0.1:27370/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140793 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140793 T489 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 140793 T489 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 140796 T489 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 140797 T489 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 140799 T489 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 140801 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 140802 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 140805 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 140805 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 140808 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 140808 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 140811 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 140811 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 140813 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 140814 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 140816 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 140816 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 140818 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 140819 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 140821 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 140821 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 140824 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 140824 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 140826 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 140827 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 140829 T489 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 140829 T489 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 142110 T489 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores/collection1
   [junit4]   2> 142114 T489 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 142118 T489 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27382
   [junit4]   2> 142118 T489 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 142119 T489 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 142119 T489 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001
   [junit4]   2> 142119 T489 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/'
   [junit4]   2> 142155 T489 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/solr.xml
   [junit4]   2> 142200 T489 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores
   [junit4]   2> 142200 T489 oasc.CoreContainer.<init> New CoreContainer 173756631
   [junit4]   2> 142201 T489 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/]
   [junit4]   2> 142202 T489 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 142202 T489 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 142202 T489 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 142203 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 142203 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 142203 T489 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 142204 T489 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 142204 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 142204 T489 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 142205 T489 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 142205 T489 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 142205 T489 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 142206 T489 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 142206 T489 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 142207 T489 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 142207 T489 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27370/solr
   [junit4]   2> 142207 T489 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 142208 T489 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 142209 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 142211 T512 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2557a5f name:ZooKeeperConnection Watcher:127.0.0.1:27370 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 142211 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 142212 T489 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 142214 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 142215 T515 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@309624c4 name:ZooKeeperConnection Watcher:127.0.0.1:27370/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 142215 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 142217 T489 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 142220 T489 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 142223 T489 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 142225 T489 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 142228 T489 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 142231 T489 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 142232 T489 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27381_h_ynf
   [junit4]   2> 142233 T489 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27381_h_ynf
   [junit4]   2> 142235 T489 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 142237 T489 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 142239 T489 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 142241 T489 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:27381_h_ynf
   [junit4]   2> 142241 T489 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 142243 T489 oasc.Overseer.start Overseer (id=93267529701720067-127.0.0.1:27381_h_ynf-n_0000000000) starting
   [junit4]   2> 142245 T489 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 142253 T489 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 142254 T517 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 142255 T489 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 142257 T489 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 142259 T489 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 142262 T516 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 142263 T489 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores
   [junit4]   2> 142265 T489 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores/collection1/
   [junit4]   2> 142265 T489 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 142267 T519 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 142267 T519 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 142268 T515 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 142268 T519 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 142271 T516 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27381_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27381/h_ynf",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 142272 T516 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:27381_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27381/h_ynf",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 142273 T516 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 142273 T516 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 142274 T515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143268 T519 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 143269 T519 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 143270 T519 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 143270 T519 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 143271 T519 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 143271 T519 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores/collection1/'
   [junit4]   2> 143285 T519 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 143288 T519 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 143300 T519 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 143322 T519 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 143368 T519 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 143370 T519 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 143386 T519 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 143866 T519 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 143883 T519 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 143886 T519 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 143912 T519 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143923 T519 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143933 T519 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143935 T519 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143936 T519 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143936 T519 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143938 T519 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143939 T519 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143939 T519 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 143940 T519 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 143940 T519 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/control/data/
   [junit4]   2> 143941 T519 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4774c9d0
   [junit4]   2> 143943 T519 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/control/data
   [junit4]   2> 143944 T519 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/control/data/index/
   [junit4]   2> 143944 T519 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 143945 T519 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/control/data/index
   [junit4]   2> 143946 T519 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=10.1181640625, floorSegmentMB=2.076171875, forceMergeDeletesPctAllowed=10.8725744331695, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4968224210713512
   [junit4]   2> 143947 T519 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e9743bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ce0559a),segFN=segments_1,generation=1}
   [junit4]   2> 143948 T519 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 143954 T519 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 143955 T519 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 143955 T519 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 143955 T519 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143956 T519 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143956 T519 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 143957 T519 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143957 T519 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143957 T519 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 143958 T519 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 143959 T519 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 143960 T519 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 143960 T519 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 143961 T519 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 143962 T519 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 143962 T519 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 143963 T519 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 143963 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 143965 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 143966 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 143967 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 143967 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 143968 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 143968 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 143969 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 143970 T519 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 143971 T519 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 143995 T519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143997 T519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144000 T519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144002 T519 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144005 T519 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 144010 T519 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 144010 T519 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 144011 T519 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=46.79296875, floorSegmentMB=1.1025390625, forceMergeDeletesPctAllowed=13.965199778237087, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052823882651978
   [junit4]   2> 144012 T519 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e9743bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ce0559a),segFN=segments_1,generation=1}
   [junit4]   2> 144012 T519 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 144013 T519 oass.SolrIndexSearcher.<init> Opening Searcher@1b09327c[collection1] main
   [junit4]   2> 144013 T519 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 144015 T519 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 144016 T519 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 144016 T519 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 144017 T519 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 144017 T519 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 144018 T519 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 144018 T519 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 144018 T519 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 144019 T519 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 144020 T520 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1b09327c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 144021 T519 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 144022 T523 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27381/h_ynf collection:control_collection shard:shard1
   [junit4]   2> 144023 T489 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 144024 T489 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 144024 T523 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 144032 T523 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 144035 T515 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 144035 T523 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 144036 T523 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C63 name=collection1 org.apache.solr.core.SolrCore@d7ee202 url=http://127.0.0.1:27381/h_ynf/collection1 node=127.0.0.1:27381_h_ynf C63_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:27381_h_ynf, state=down, base_url=http://127.0.0.1:27381/h_ynf, core=collection1}
   [junit4]   2> 144036 T523 C63 P27381 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27381/h_ynf/collection1/
   [junit4]   2> 144036 T523 C63 P27381 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 144036 T516 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 144037 T523 C63 P27381 oasc.SyncStrategy.syncToMe http://127.0.0.1:27381/h_ynf/collection1/ has no replicas
   [junit4]   2> 144037 T523 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27381/h_ynf/collection1/ shard1
   [junit4]   2> 144038 T523 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 144039 T515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 144046 T524 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 144047 T516 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27381/h_ynf",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 144049 T489 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 144050 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 144052 T528 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aff2cd8 name:ZooKeeperConnection Watcher:127.0.0.1:27370/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 144053 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 144054 T489 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 144056 T489 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 144060 T489 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 144154 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 144199 T523 oasc.ZkController.register We are http://127.0.0.1:27381/h_ynf/collection1/ and leader is http://127.0.0.1:27381/h_ynf/collection1/
   [junit4]   2> 144200 T523 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27381/h_ynf
   [junit4]   2> 144200 T523 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 144200 T523 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 144201 T523 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 144203 T515 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 144206 T516 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:27381_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27381/h_ynf",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 144207 T516 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:27381_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27381/h_ynf",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 144314 T515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 144314 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 146084 T489 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores/collection1
   [junit4]   2> 146087 T489 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 146088 T489 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 146091 T489 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27402
   [junit4]   2> 146092 T489 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 146092 T489 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 146093 T489 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001
   [junit4]   2> 146093 T489 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/'
   [junit4]   2> 146144 T489 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/solr.xml
   [junit4]   2> 146205 T489 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores
   [junit4]   2> 146206 T489 oasc.CoreContainer.<init> New CoreContainer 521044906
   [junit4]   2> 146206 T489 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/]
   [junit4]   2> 146208 T489 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 146208 T489 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 146208 T489 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 146209 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 146209 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 146209 T489 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 146210 T489 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 146210 T489 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 146210 T489 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 146211 T489 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 146211 T489 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 146212 T489 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 146212 T489 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 146213 T489 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 146213 T489 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 146213 T489 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27370/solr
   [junit4]   2> 146214 T489 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 146214 T489 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 146215 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146217 T540 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@755026db name:ZooKeeperConnection Watcher:127.0.0.1:27370 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146218 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146218 T489 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 146221 T489 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146223 T543 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f0e5014 name:ZooKeeperConnection Watcher:127.0.0.1:27370/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146223 T489 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146231 T489 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 147236 T489 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27401_h_ynf
   [junit4]   2> 147238 T489 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27401_h_ynf
   [junit4]   2> 147242 T489 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 147246 T489 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores
   [junit4]   2> 147249 T489 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores/collection1/
   [junit4]   2> 147249 T489 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 147251 T544 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 147251 T544 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 147252 T515 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 147252 T544 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 147254 T516 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:27401_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27401/h_ynf",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 147255 T516 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:27401_h_ynf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27401/h_ynf",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 147255 T516 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 147255 T516 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 147257 T515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 147257 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 147257 T543 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 148253 T544 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 148254 T544 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 148255 T544 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 148255 T544 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 148256 T544 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 148256 T544 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores/collection1/'
   [junit4]   2> 148273 T544 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 148276 T544 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 148290 T544 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 148315 T544 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 148368 T544 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 148370 T544 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 148418 T544 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 148910 T544 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 148925 T544 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 148929 T544 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 148952 T544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 148959 T544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 148965 T544 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 148966 T544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 148967 T544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 148967 T544 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 148968 T544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 148969 T544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 148969 T544 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 148969 T544 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 148970 T544 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/jetty1/
   [junit4]   2> 148970 T544 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4774c9d0
   [junit4]   2> 148971 T544 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/jetty1
   [junit4]   2> 148971 T544 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/jetty1/index/
   [junit4]   2> 148972 T544 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7BA0F00E400B55A8-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 148972 T544 oasc.

[...truncated too long message...]

cene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10/index.20150205103513025,segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10/index.20150205103513025,segFN=segments_5,generation=5}
   [junit4]   2> 1448233 T4075 C774 P51134 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 1448275 T4075 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1448278 T4075 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1448278 T4075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10;done=false>>]
   [junit4]   2> 1448279 T4075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10
   [junit4]   2> 1448279 T4075 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10/index.20150205103513025 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10/index.20150205103513025;done=false>>]
   [junit4]   2> 1448279 T4075 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001/tempDir-001/jetty10/index.20150205103513025
   [junit4]   2> 1448281 T4075 oasc.Overseer.close Overseer (id=93267587174760479-127.0.0.1:51134_-n_0000000013) closing
   [junit4]   2> 1448282 T4596 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:51134_
   [junit4]   2> 1449785 T4594 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1449788 T4075 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1449983 T4075 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50870 50870
   [junit4]   2> 1450053 T4076 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50870 50870
   [junit4]   2> 1450055 T4076 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		16	/solr/clusterstate.json
   [junit4]   2> 		16	/solr/aliases.json
   [junit4]   2> 		15	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		3	/solr/overseer_elect/election/93267587174760465-127.0.0.1:51068_-n_0000000006
   [junit4]   2> 		2	/solr/overseer_elect/election/93267587174760467-127.0.0.1:51081_-n_0000000007
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard4/election/93267587174760459-core_node3-n_0000000000
   [junit4]   2> 		2	/solr/overseer_elect/election/93267587174760475-127.0.0.1:51101_-n_0000000011
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93267587174760461-core_node4-n_0000000000
   [junit4]   2> 		2	/solr/overseer_elect/election/93267587174760471-127.0.0.1:51119_-n_0000000009
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard2/election/93267587174760465-core_node6-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard3/election/93267587174760463-core_node5-n_0000000001
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		12	/solr/live_nodes
   [junit4]   2> 		9	/solr/overseer/queue
   [junit4]   2> 		9	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=test -Dtests.seed=7BA0F00E400B55A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_EC -Dtests.timezone=America/Indianapolis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  433s J0 | ChaosMonkeySafeLeaderTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<0> but was:<1>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7BA0F00E400B55A8:F3F4CFD4EEF73850]:0)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:152)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1450065 T4075 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ChaosMonkeySafeLeaderTest 7BA0F00E400B55A8-001
   [junit4]   2> 432586 T4074 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=Lucene50(blocksize=128), a_i=PostingsFormat(name=Asserting), id=PostingsFormat(name=Asserting), a_t=Lucene50(blocksize=128), rnd_b=PostingsFormat(name=Asserting)}, docValues:{}, sim=DefaultSimilarity, locale=es_EC, timezone=America/Indianapolis
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=95969312,total=381157376
   [junit4]   2> NOTE: All tests run in this JVM: [SolrIndexConfigTest, HighlighterMaxOffsetTest, TestCollectionAPI, SharedFSAutoReplicaFailoverTest, DistributedFacetPivotLongTailTest, SolrCmdDistributorTest, ShardRoutingCustomTest, TestManagedSchemaDynamicFieldResource, ShowFileRequestHandlerTest, ZkNodePropsTest, TestSchemaResource, AtomicUpdatesTest, MultiThreadedOCPTest, TestRandomFaceting, DebugComponentTest, TestIBSimilarityFactory, TestSort, RecoveryZkTest, IndexBasedSpellCheckerTest, TestCloudInspectUtil, TestCollationField, OverseerCollectionProcessorTest, DistributedSpellCheckComponentTest, ResponseHeaderTest, TestCharFilters, TestSolrConfigHandlerConcurrent, DistributedQueryElevationComponentTest, RequestHandlersTest, TestOrdValues, TestDocSet, DistributedExpandComponentTest, MinimalSchemaTest, TestMiniSolrCloudCluster, SuggesterTest, SpellCheckCollatorTest, DistribCursorPagingTest, SimpleFacetsTest, TestReRankQParserPlugin, TestCollapseQParserPlugin, SearchHandlerTest, TestClassNameShortening, TestReloadAndDeleteDocs, OpenExchangeRatesOrgProviderTest, TestFieldTypeResource, HdfsWriteToMultipleCollectionsTest, TestManagedSchemaFieldResource, LeaderFailoverAfterPartitionTest, TestShortCircuitedRequests, TestDFRSimilarityFactory, BasicFunctionalityTest, ExternalFileFieldSortTest, TimeZoneUtilsTest, TestComplexPhraseQParserPlugin, TestManagedResource, TestCopyFieldCollectionResource, AnalyticsMergeStrategyTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J0 in 433.13s, 1 test, 1 failure <<< FAILURES!

[...truncated 837 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1852 tests, 5 errors, 1 failure, 56 ignored (9 assumptions)

Total time: 152 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 689172247 bytes
Compression is 0.0%
Took 2 min 7 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 753 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/753/

6 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1423064611152,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_Lucene50_0.doc, _0_Lucene50_0.tim, _0_Lucene50_0.tip, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_Lucene50_0.doc, _6_Lucene50_0.tim, _6_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, segments_2]}]> but was:<[{indexVersion=1423064611152,generation=3,filelist=[_4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_3]}, {indexVersion=1423064611152,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_Lucene50_0.doc, _0_Lucene50_0.tim, _0_Lucene50_0.tip, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_Lucene50_0.doc, _6_Lucene50_0.tim, _6_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1423064611152,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_Lucene50_0.doc, _0_Lucene50_0.tim, _0_Lucene50_0.tip, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_Lucene50_0.doc, _6_Lucene50_0.tim, _6_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, segments_2]}]> but was:<[{indexVersion=1423064611152,generation=3,filelist=[_4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_3]}, {indexVersion=1423064611152,generation=2,filelist=[_0.fdt, _0.fdx, _0.fnm, _0.nvd, _0.nvm, _0.si, _0_Lucene50_0.doc, _0_Lucene50_0.tim, _0_Lucene50_0.tip, _4.fdt, _4.fdx, _4.fnm, _4.nvd, _4.nvm, _4.si, _4_Lucene50_0.doc, _4_Lucene50_0.tim, _4_Lucene50_0.tip, _6.fdt, _6.fdx, _6.fnm, _6.nvd, _6.nvm, _6.si, _6_Lucene50_0.doc, _6_Lucene50_0.tim, _6_Lucene50_0.tip, _7.cfe, _7.cfs, _7.si, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:76C54E413D17EBB0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:45537/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45537/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9653 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/init-core-data-001
   [junit4]   2> 65131 T150 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 65176 T150 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 65184 T151 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 65485 T150 oasc.ZkTestServer.run start zk server on port:59096
   [junit4]   2> 65644 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 65901 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66081 T158 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@753fc41d name:ZooKeeperConnection Watcher:127.0.0.1:59096 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66082 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66083 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66091 T150 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 66186 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 66187 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66189 T161 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@658eeb10 name:ZooKeeperConnection Watcher:127.0.0.1:59096/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66190 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66190 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66197 T150 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 66202 T150 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 66207 T150 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 66211 T150 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 66215 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 66240 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 66247 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 66248 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 66252 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66253 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66257 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 66258 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 66262 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 66262 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 66266 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 66267 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 66271 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 66271 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 66276 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66276 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66280 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66281 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66285 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 66286 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 66290 T150 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 66291 T150 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 67790 T150 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores/collection1
   [junit4]   2> 67796 T150 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 67801 T150 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63251
   [junit4]   2> 67802 T150 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 67802 T150 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 67803 T150 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001
   [junit4]   2> 67803 T150 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/'
   [junit4]   2> 67868 T150 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/solr.xml
   [junit4]   2> 67945 T150 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores
   [junit4]   2> 67946 T150 oasc.CoreContainer.<init> New CoreContainer 455596359
   [junit4]   2> 67947 T150 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/]
   [junit4]   2> 67949 T150 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 67949 T150 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 67950 T150 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 67950 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 67951 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 67951 T150 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 67952 T150 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 67952 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 67952 T150 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 67953 T150 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 67953 T150 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 67954 T150 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 67955 T150 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 67955 T150 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 67956 T150 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 67956 T150 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59096/solr
   [junit4]   2> 68043 T150 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 68044 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 68045 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68048 T173 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b6b4262 name:ZooKeeperConnection Watcher:127.0.0.1:59096 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68048 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68049 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 68069 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 68071 T176 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b5d2b0b name:ZooKeeperConnection Watcher:127.0.0.1:59096/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 68071 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 68094 T150 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 68111 T150 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 68122 T150 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 68126 T150 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 68130 T150 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 68155 T150 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 68157 T150 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57360_
   [junit4]   2> 68167 T150 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57360_
   [junit4]   2> 68171 T150 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 68174 T150 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 68177 T150 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 68203 T150 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57360_
   [junit4]   2> 68204 T150 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 68206 T150 oasc.Overseer.start Overseer (id=93261907615547395-127.0.0.1:57360_-n_0000000000) starting
   [junit4]   2> 68232 T150 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 68270 T150 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 68295 T178 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 68296 T150 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 68299 T150 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 68302 T150 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 68340 T177 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 68340 T150 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores
   [junit4]   2> 68365 T150 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores/collection1/
   [junit4]   2> 68365 T150 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 68367 T180 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 68368 T180 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 68388 T176 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 68388 T180 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 68402 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:57360_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57360",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 68449 T177 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:57360_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57360",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 68456 T177 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 68468 T177 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 68487 T176 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 69390 T180 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 69390 T180 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 69392 T180 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 69392 T180 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 69394 T180 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 69394 T180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores/collection1/'
   [junit4]   2> 69456 T180 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 69483 T180 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 69519 T180 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 69552 T180 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 69610 T180 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 69613 T180 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 69629 T180 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 70110 T180 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 70125 T180 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 70129 T180 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 70156 T180 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70163 T180 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70169 T180 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70172 T180 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70172 T180 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70172 T180 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70174 T180 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70175 T180 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70175 T180 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 70176 T180 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 70176 T180 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/tempDir-001/control/data/
   [junit4]   2> 70177 T180 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62343903
   [junit4]   2> 70193 T180 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/tempDir-001/control/data
   [junit4]   2> 70194 T180 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/tempDir-001/control/data/index/
   [junit4]   2> 70194 T180 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 70195 T180 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/tempDir-001/control/data/index
   [junit4]   2> 70195 T180 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5194580902445621]
   [junit4]   2> 70197 T180 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f7714c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72f58fa3),segFN=segments_1,generation=1}
   [junit4]   2> 70197 T180 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70208 T180 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 70208 T180 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 70209 T180 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 70209 T180 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70210 T180 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70210 T180 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 70211 T180 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70211 T180 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70212 T180 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 70222 T180 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70223 T180 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70224 T180 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70225 T180 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70227 T180 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 70228 T180 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 70229 T180 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 70230 T180 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 70232 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 70234 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 70235 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 70236 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 70237 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 70239 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 70240 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 70241 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 70242 T180 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 70244 T180 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 70266 T180 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70268 T180 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70271 T180 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70273 T180 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70276 T180 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 70280 T180 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 70280 T180 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 70282 T180 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=0.2421875, floorSegmentMB=0.52734375, forceMergeDeletesPctAllowed=18.79729122203152, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 70283 T180 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f7714c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72f58fa3),segFN=segments_1,generation=1}
   [junit4]   2> 70283 T180 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70284 T180 oass.SolrIndexSearcher.<init> Opening Searcher@7379e27[collection1] main
   [junit4]   2> 70284 T180 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 70286 T180 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 70287 T180 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 70299 T180 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 70300 T180 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 70300 T180 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70301 T180 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 70302 T180 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70302 T180 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 70303 T180 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 70306 T181 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7379e27[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 70313 T180 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 70314 T184 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57360 collection:control_collection shard:shard1
   [junit4]   2> 70315 T150 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 70316 T150 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 70352 T184 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 70364 T184 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 70367 T176 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70368 T184 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 70368 T184 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C23 name=collection1 org.apache.solr.core.SolrCore@1eaa58ff url=http://127.0.0.1:57360/collection1 node=127.0.0.1:57360_ C23_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:57360_, base_url=http://127.0.0.1:57360, state=down, core=collection1}
   [junit4]   2> 70369 T184 C23 P57360 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57360/collection1/
   [junit4]   2> 70369 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 70370 T184 C23 P57360 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 70370 T184 C23 P57360 oasc.SyncStrategy.syncToMe http://127.0.0.1:57360/collection1/ has no replicas
   [junit4]   2> 70371 T184 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57360/collection1/ shard1
   [junit4]   2> 70378 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 70380 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70382 T188 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aed393b name:ZooKeeperConnection Watcher:127.0.0.1:59096/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70383 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70384 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 70386 T176 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70386 T150 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 70386 T184 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 70397 T189 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70397 T150 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 70398 T150 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 70399 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57360",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 70423 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 70424 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70427 T192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4adb618 name:ZooKeeperConnection Watcher:127.0.0.1:59096/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70427 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70428 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 70432 T189 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70434 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 70434 T177 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 70436 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70546 T177 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 70549 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70603 T184 oasc.ZkController.register We are http://127.0.0.1:57360/collection1/ and leader is http://127.0.0.1:57360/collection1/
   [junit4]   2> 70603 T184 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57360
   [junit4]   2> 70603 T184 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 70604 T184 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 70604 T184 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 70607 T176 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70611 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:57360_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57360",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 4
   [junit4]   2> 70612 T177 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:57360_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57360",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 70719 T188 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70719 T176 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71819 T150 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/cores/collection1
   [junit4]   2> 71822 T150 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 71822 T150 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 71825 T150 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59223
   [junit4]   2> 71825 T150 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 71826 T150 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 71826 T150 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001
   [junit4]   2> 71826 T150 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/'
   [junit4]   2> 71860 T150 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/solr.xml
   [junit4]   2> 71901 T150 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/cores
   [junit4]   2> 71902 T150 oasc.CoreContainer.<init> New CoreContainer 1080038042
   [junit4]   2> 71902 T150 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/]
   [junit4]   2> 71903 T150 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 71903 T150 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 71904 T150 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 71904 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 71904 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 71905 T150 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 71905 T150 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 71905 T150 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 71905 T150 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 71906 T150 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 71906 T150 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 71907 T150 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 71907 T150 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 71907 T150 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 71908 T150 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 71908 T150 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59096/solr
   [junit4]   2> 71908 T150 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 71909 T150 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 71909 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 71911 T204 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70029fd0 name:ZooKeeperConnection Watcher:127.0.0.1:59096 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 71912 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 71913 T150 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 71915 T150 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 71916 T207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c5d2aea name:ZooKeeperConnection Watcher:127.0.0.1:59096/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 71917 T150 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 71924 T150 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 72931 T150 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21910_
   [junit4]   2> 72933 T150 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21910_
   [junit4]   2> 72937 T150 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 72945 T150 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/cores
   [junit4]   2> 72947 T150 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/cores/collection1/
   [junit4]   2> 72948 T150 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 72952 T208 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 72952 T208 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 72955 T176 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 72956 T208 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 72957 T208 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 72957 T177 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:21910_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21910",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 72957 T208 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 72958 T177 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:21910_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21910",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 72959 T177 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 72959 T177 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 72962 T177 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 72966 T208 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 72966 T208 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 72967 T208 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 72967 T208 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 72968 T208 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 72968 T208 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 72969 T208 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 72970 T208 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 531255714D5FE5B3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 72987 T208 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 72991 T208 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 73004 T208 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 73028 T208 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 73089 T208 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 73091 T208 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 73106 T208 oass.IndexSchema.readSchema [collection1] Schema name=

[...truncated too long message...]

t4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1876674 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45523/
   [junit4]   2> 1876675 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45588/
   [junit4]   2> 1876675 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45531/
   [junit4]   2> 1876675 T10889 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:45537/
   [junit4]   2> 1876675 T10889 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:45569/
   [junit4]   2> 1876676 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45523/
   [junit4]   2> 1876676 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45588/
   [junit4]   2> 1876677 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45531/
   [junit4]   2> 1876677 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45537/
   [junit4]   2> 1876677 T10889 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45569/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=531255714D5FE5B3 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_TW -Dtests.timezone=Atlantic/Canary -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   74.6s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45537/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([531255714D5FE5B3:DB466AABE3A3884B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 49 more
   [junit4]   2> 1876688 T10889 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 531255714D5FE5B3-001
   [junit4]   2> 74589 T10888 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=zh_TW, timezone=Atlantic/Canary
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=150720088,total=396886016
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecovery, HdfsBasicDistributedZkTest, RequestLoggingTest, DistributedQueryComponentCustomSortTest, AssignTest, RecoveryAfterSoftCommitTest, TestZkChroot, IndexSchemaTest, TestClusterStateMutator, ChaosMonkeySafeLeaderTest, OverseerStatusTest, DistributedFacetPivotLargeTest, QueryElevationComponentTest, RollingRestartTest, DistributedQueryComponentOptimizationTest, TestSolrDeletionPolicy1, FieldAnalysisRequestHandlerTest, AutoCommitTest, UpdateParamsTest, TestLFUCache, CursorPagingTest, TestFunctionQuery, DistributedDebugComponentTest, BasicZkTest, HighlighterTest, TestDefaultSearchFieldResource, HdfsUnloadDistributedZkTest, UniqFieldsUpdateProcessorFactoryTest, SuggesterFSTTest, TestSweetSpotSimilarityFactory, TestDynamicFieldResource, WordBreakSolrSpellCheckerTest, FullSolrCloudDistribCmdsTest, AddBlockUpdateTest, TestStressRecovery, BJQParserTest, JSONWriterTest, MergeStrategyTest, EchoParamsTest, DocumentAnalysisRequestHandlerTest, ZkControllerTest, PingRequestHandlerTest, TestOmitPositions, TestSolrQueryParserDefaultOperatorResource, DistanceUnitsTest, HttpPartitionTest, FieldMutatingUpdateProcessorTest, ConnectionManagerTest, TestFastOutputStream, TestBadConfig, DisMaxRequestHandlerTest, SystemInfoHandlerTest, TestRecoveryHdfs, QueryEqualityTest, DocValuesMultiTest, DistanceFunctionTest, TestSurroundQueryParser, TestQueryUtils, TestReversedWildcardFilterFactory, FileBasedSpellCheckerTest, DocumentBuilderTest, TermVectorComponentTest, LoggingHandlerTest, ReturnFieldsTest, QueryParsingTest, TestPHPSerializedResponseWriter, TestQuerySenderNoQuery, SolrIndexConfigTest, TestSearchPerf, OutputWriterTest, TestSolrCoreProperties, TestPhraseSuggestions, TestDefaultSimilarityFactory, PluginInfoTest, PreAnalyzedFieldTest, DOMUtilTest, ClusterStateTest, SliceStateTest, CircularListTest, DistributedIntervalFacetingTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 74.87s, 1 test, 1 error <<< FAILURES!

[...truncated 409 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1849 tests, 5 errors, 1 failure, 56 ignored (9 assumptions)

Total time: 148 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 687888661 bytes
Compression is 0.0%
Took 2 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 752 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/752/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44424/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44424/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44424/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43661/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43661/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43661/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:22069/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22069/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22371/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22371/collection1
	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22371/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22425/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22425/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22425/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9873 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/init-core-data-001
   [junit4]   2> 184889 T935 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 184895 T935 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 184895 T936 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 184995 T935 oasc.ZkTestServer.run start zk server on port:43624
   [junit4]   2> 184996 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 184997 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 185003 T943 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@619db33e name:ZooKeeperConnection Watcher:127.0.0.1:43624 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 185003 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 185004 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 185004 T935 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 185007 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 185008 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 185009 T946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66321800 name:ZooKeeperConnection Watcher:127.0.0.1:43624/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 185010 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 185010 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 185010 T935 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 185013 T935 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 185015 T935 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 185016 T935 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 185019 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 185020 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 185023 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 185023 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 185025 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 185026 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 185028 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 185028 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 185030 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 185031 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 185033 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 185033 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 185035 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 185035 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 185037 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 185038 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 185040 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 185041 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 185043 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 185043 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 185045 T935 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 185046 T935 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 186448 T935 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores/collection1
   [junit4]   2> 186452 T935 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 186456 T935 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43630
   [junit4]   2> 186457 T935 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 186457 T935 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 186457 T935 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001
   [junit4]   2> 186458 T935 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/'
   [junit4]   2> 186501 T935 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/solr.xml
   [junit4]   2> 186555 T935 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores
   [junit4]   2> 186556 T935 oasc.CoreContainer.<init> New CoreContainer 2052795162
   [junit4]   2> 186556 T935 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/]
   [junit4]   2> 186558 T935 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 186558 T935 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 186559 T935 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 186559 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 186559 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 186560 T935 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 186560 T935 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 186561 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 186561 T935 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 186561 T935 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 186562 T935 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 186562 T935 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 186563 T935 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 186563 T935 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 186564 T935 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 186564 T935 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43624/solr
   [junit4]   2> 186565 T935 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 186565 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 186566 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186568 T958 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2501dde2 name:ZooKeeperConnection Watcher:127.0.0.1:43624 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186569 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186570 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 186572 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186574 T961 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@59370c7e name:ZooKeeperConnection Watcher:127.0.0.1:43624/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186574 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186576 T935 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 186579 T935 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 186581 T935 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 186584 T935 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 186586 T935 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 186589 T935 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 186591 T935 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43629_
   [junit4]   2> 186592 T935 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43629_
   [junit4]   2> 186595 T935 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 186597 T935 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 186598 T935 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 186600 T935 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:43629_
   [junit4]   2> 186601 T935 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 186603 T935 oasc.Overseer.start Overseer (id=93256277422309379-127.0.0.1:43629_-n_0000000000) starting
   [junit4]   2> 186605 T935 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 186613 T935 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 186614 T963 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 186614 T935 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 186617 T935 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 186618 T935 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 186622 T962 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 186623 T935 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores
   [junit4]   2> 186626 T935 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores/collection1/
   [junit4]   2> 186626 T935 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 186627 T965 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 186628 T965 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 186629 T961 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 186629 T965 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 186632 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43629",
   [junit4]   2> 	  "node_name":"127.0.0.1:43629_"} current state version: 0
   [junit4]   2> 186633 T962 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43629",
   [junit4]   2> 	  "node_name":"127.0.0.1:43629_"}
   [junit4]   2> 186633 T962 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 186633 T962 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 186635 T961 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 187629 T965 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 187630 T965 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 187631 T965 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 187631 T965 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 187632 T965 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 187632 T965 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores/collection1/'
   [junit4]   2> 187646 T965 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 187649 T965 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 187661 T965 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 187681 T965 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 187721 T965 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 187723 T965 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 187744 T965 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 188156 T965 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 188169 T965 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 188172 T965 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 188186 T965 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188191 T965 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188196 T965 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188197 T965 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188198 T965 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188198 T965 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188200 T965 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188200 T965 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188200 T965 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 188201 T965 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 188201 T965 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/control/data/
   [junit4]   2> 188201 T965 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@65f673ad
   [junit4]   2> 188204 T965 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/control/data
   [junit4]   2> 188205 T965 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/control/data/index/
   [junit4]   2> 188206 T965 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 188206 T965 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/control/data/index
   [junit4]   2> 188207 T965 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=89.251953125, floorSegmentMB=2.078125, forceMergeDeletesPctAllowed=0.9421940229109671, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5623828423246596
   [junit4]   2> 188208 T965 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@30640db6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@78c0408b),segFN=segments_1,generation=1}
   [junit4]   2> 188208 T965 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188214 T965 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 188214 T965 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 188215 T965 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 188215 T965 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 188215 T965 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 188216 T965 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 188216 T965 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 188217 T965 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 188217 T965 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 188218 T965 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 188219 T965 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 188219 T965 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 188220 T965 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 188220 T965 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 188221 T965 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 188222 T965 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 188222 T965 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 188223 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 188224 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 188224 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 188225 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 188226 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 188226 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 188227 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 188227 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 188228 T965 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 188229 T965 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 188251 T965 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188253 T965 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188257 T965 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188259 T965 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188261 T965 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 188266 T965 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 188266 T965 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 188267 T965 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=10.0576171875, floorSegmentMB=0.7578125, forceMergeDeletesPctAllowed=20.841755242340472, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 188268 T965 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@30640db6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@78c0408b),segFN=segments_1,generation=1}
   [junit4]   2> 188268 T965 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188269 T965 oass.SolrIndexSearcher.<init> Opening Searcher@74c512da[collection1] main
   [junit4]   2> 188269 T965 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 188270 T965 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 188270 T965 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 188271 T965 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 188271 T965 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 188272 T965 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188272 T965 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 188272 T965 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188273 T965 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 188273 T965 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 188274 T966 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@74c512da[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 188275 T965 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 188275 T969 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43629 collection:control_collection shard:shard1
   [junit4]   2> 188276 T935 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 188276 T935 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 188276 T969 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 188280 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 188281 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 188282 T973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aaed9a9 name:ZooKeeperConnection Watcher:127.0.0.1:43624/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 188283 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 188283 T969 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 188283 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 188285 T935 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 188285 T961 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188286 T969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 188286 T969 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C80 name=collection1 org.apache.solr.core.SolrCore@463b0723 url=http://127.0.0.1:43629/collection1 node=127.0.0.1:43629_ C80_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:43629, node_name=127.0.0.1:43629_}
   [junit4]   2> 188286 T969 C80 P43629 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43629/collection1/
   [junit4]   2> 188286 T969 C80 P43629 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 188286 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 188286 T969 C80 P43629 oasc.SyncStrategy.syncToMe http://127.0.0.1:43629/collection1/ has no replicas
   [junit4]   2> 188287 T969 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43629/collection1/ shard1
   [junit4]   2> 188287 T969 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 188288 T935 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 188288 T973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188288 T935 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 188288 T961 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188289 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 188290 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 188291 T976 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@744556d name:ZooKeeperConnection Watcher:127.0.0.1:43624/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 188292 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 188292 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 188293 T977 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188294 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43629",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 188297 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 188298 T962 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 188298 T973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188301 T977 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188302 T962 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 188303 T973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188344 T969 oasc.ZkController.register We are http://127.0.0.1:43629/collection1/ and leader is http://127.0.0.1:43629/collection1/
   [junit4]   2> 188345 T969 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43629
   [junit4]   2> 188345 T969 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 188345 T969 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 188345 T969 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 188347 T961 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188348 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43629",
   [junit4]   2> 	  "node_name":"127.0.0.1:43629_",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 4
   [junit4]   2> 188349 T962 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43629",
   [junit4]   2> 	  "node_name":"127.0.0.1:43629_",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 188453 T961 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188453 T973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 189701 T935 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores/collection1
   [junit4]   2> 189703 T935 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 189704 T935 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 189706 T935 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43662
   [junit4]   2> 189707 T935 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 189707 T935 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 189707 T935 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001
   [junit4]   2> 189708 T935 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/'
   [junit4]   2> 189737 T935 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/solr.xml
   [junit4]   2> 189771 T935 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores
   [junit4]   2> 189772 T935 oasc.CoreContainer.<init> New CoreContainer 1874661130
   [junit4]   2> 189772 T935 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/]
   [junit4]   2> 189773 T935 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 189773 T935 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 189773 T935 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 189773 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 189774 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 189774 T935 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 189774 T935 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 189774 T935 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 189775 T935 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 189775 T935 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 189775 T935 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 189776 T935 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 189776 T935 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 189776 T935 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 189777 T935 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 189777 T935 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43624/solr
   [junit4]   2> 189777 T935 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 189777 T935 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 189778 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 189780 T989 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78e4f765 name:ZooKeeperConnection Watcher:127.0.0.1:43624 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 189780 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 189781 T935 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 189782 T935 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 189783 T992 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@325f52f0 name:ZooKeeperConnection Watcher:127.0.0.1:43624/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 189784 T935 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 189788 T935 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 190792 T935 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43661_
   [junit4]   2> 190793 T935 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43661_
   [junit4]   2> 190797 T935 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 190800 T935 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores
   [junit4]   2> 190803 T935 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores/collection1/
   [junit4]   2> 190803 T935 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 190806 T993 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 190806 T993 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 190807 T961 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 190808 T993 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 190808 T993 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 190809 T962 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43661",
   [junit4]   2> 	  "node_name":"127.0.0.1:43661_"} current state version: 5
   [junit4]   2> 190809 T993 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 190809 T962 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43661",
   [junit4]   2> 	  "node_name":"127.0.0.1:43661_"}
   [junit4]   2> 190810 T962 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 190810 T962 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 190810 T993 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 190810 T993 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 190812 T962 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 190812 T992 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 190813 T992 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 191811 T993 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 191812 T993 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 191812 T993 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 191813 T993 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 191813 T993 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 191813 T993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores/collection1/'
   [junit4]   2> 191826 T993 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 191828 T993 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 191838 T993 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 191871 T993 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 191908 T993 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 191909 T993 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 191928 T993 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 192235 T993 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 192245 T993 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 192247 T993 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 192260 T993 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 192264 T993 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 192269 T993 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 192270 T993 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 192271 T993 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 192271 T993 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 192272 T993 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 192272 T993 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 192273 T993 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 192273 T993 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 192273 T993 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/jetty1/
   [junit4]   2> 192274 T993 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@65f673ad
   [junit4]   2> 192276 T993 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/jetty1
   [junit4]   2> 192277 T993 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/jetty1/index/
   [junit4]   2> 192277 T993 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 192277 T993 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest C34D9A8EFC4CA1CE-001/tempDir-001/jetty1/index
   [junit4]   2> 192278 T993 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=89.251953125, floorSegmentMB=2.078125, forceMergeDeletesPctAllowed=0.9421940229109671, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5623828423246596
   [junit4]   2> 192278 T993 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(RAMDirectory@aabcea5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@158bb81c),segFN=segments_1,generation=1}
   [junit4]   2> 192279 T993 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 192284 T993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 192284 T993 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 192285 T993 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcesso

[...truncated too long message...]

with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=C34D9A8EFC4CA1CE -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_HK -Dtests.timezone=Asia/Colombo -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   73.1s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22425/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C34D9A8EFC4CA1CE:4B19A55452B0CC36]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22425/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 1921709 T6283 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C34D9A8EFC4CA1CE-001
   [junit4]   2> 73088 T6282 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=30177, maxDocsPerChunk=6, blockSize=9), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=30177, blockSize=9)), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=zh_HK, timezone=Asia/Colombo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=79058800,total=326107136
   [junit4]   2> NOTE: All tests run in this JVM: [TestStandardQParsers, QueryParsingTest, HighlighterMaxOffsetTest, ZkNodePropsTest, TestDocSet, TestReplicaProperties, ExternalFileFieldSortTest, TestFaceting, TestSchemaVersionResource, TestCopyFieldCollectionResource, AtomicUpdatesTest, DirectSolrSpellCheckerTest, DOMUtilTest, TestSolrJ, TestBlendedInfixSuggestions, TestUpdate, TestCursorMarkWithoutUniqueKey, TimeZoneUtilsTest, TestManagedSchemaFieldResource, TestCloudManagedSchema, MBeansHandlerTest, DistributedQueryElevationComponentTest, DistributedSpellCheckComponentTest, TestXIncludeConfig, LeaderFailoverAfterPartitionTest, IndexBasedSpellCheckerTest, TestFastLRUCache, TestSolrConfigHandlerConcurrent, TestSchemaResource, TestCollectionAPI, HdfsWriteToMultipleCollectionsTest, TestExactSharedStatsCache, RegexBoostProcessorTest, TestManagedSchemaDynamicFieldResource, TestMiniSolrCloudCluster, DistributedFacetPivotSmallTest, OverseerTest, VMParamsZkACLAndCredentialsProvidersTest, FastVectorHighlighterTest, TestRandomFaceting, TestCloudInspectUtil, BadCopyFieldTest, TestIBSimilarityFactory, ZkSolrClientTest, TestReRankQParserPlugin, ExternalCollectionsTest, TestCollationField, TestOverriddenPrefixQueryForCustomFieldType, HighlighterConfigTest, SaslZkACLProviderTest, RequiredFieldsTest, TestCollapseQParserPlugin, SignatureUpdateProcessorFactoryTest, TestOrdValues, TestClassNameShortening, TestDocBasedVersionConstraints, SyncSliceTest, BasicZkTest, TestZkChroot, ShardRoutingCustomTest, TestDistributedGrouping, TestJoin, HardAutoCommitTest, TestCoreContainer, SimpleFacetsTest, StatsComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, TestSort, OverseerCollectionProcessorTest, SoftAutoCommitTest, CoreAdminHandlerTest, SuggesterTest, SpatialFilterTest, PolyFieldTest, QueryEqualityTest, DefaultValueUpdateProcessorTest, DistanceFunctionTest, CacheHeaderTest, TestWriterPerf, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, CSVRequestHandlerTest, SearchHandlerTest, SOLR749Test, ResponseLogComponentTest, TestSolrDeletionPolicy2, NumericFieldsTest, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, EchoParamsTest, TestDFRSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestJmxMonitoredMap, ResourceLoaderTest, URLClassifyProcessorTest, ChaosMonkeyNothingIsSafeTest, MigrateRouteKeyTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 73.37s, 1 test, 1 error <<< FAILURES!

[...truncated 278 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1849 tests, 5 errors, 56 ignored (9 assumptions)

Total time: 166 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 681381900 bytes
Compression is 0.0%
Took 2 min 7 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 751 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/751/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48789/vl/o/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48789/vl/o/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48789/vl/o/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53167/u_mst/df/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53167/u_mst/df/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53167/u_mst/df/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:64448/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64448/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63360/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63360/collection1
	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63360/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39879/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39879/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39879/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9524 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> log4j:WARN No such property [conversionPattern] in org.apache.solr.util.SolrLogLayout.
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/init-core-data-001
   [junit4]   2> 5864 T11 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 6065 T11 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 6077 T12 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 6580 T11 oasc.ZkTestServer.run start zk server on port:33816
   [junit4]   2> 6777 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 7083 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 7351 T19 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d307ef8 name:ZooKeeperConnection Watcher:127.0.0.1:33816 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 7353 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 7366 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 7378 T11 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 7516 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 7518 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 7522 T22 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5658afe1 name:ZooKeeperConnection Watcher:127.0.0.1:33816/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 7524 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 7525 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 7546 T11 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 7553 T11 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 7559 T11 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 7563 T11 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 7569 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 7637 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 7647 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 7648 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 7655 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 7656 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 7663 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 7664 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 7669 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 7670 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 7676 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 7677 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 7683 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 7685 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 7691 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 7692 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 7698 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 7699 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 7705 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 7707 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 7713 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 7714 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 10382 T11 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores/collection1
   [junit4]   2> 11622 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 12135 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42613
   [junit4]   2> 12663 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 12833 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 12834 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001
   [junit4]   2> 12835 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/'
   [junit4]   2> 14236 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/solr.xml
   [junit4]   2> 14599 T11 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores
   [junit4]   2> 14656 T11 oasc.CoreContainer.<init> New CoreContainer 2081028510
   [junit4]   2> 14657 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/]
   [junit4]   2> 14926 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 14927 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 14941 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 14941 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 14942 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 14943 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 14943 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 14944 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 14944 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 14945 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 14946 T11 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 15672 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 15701 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 15710 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 15724 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 15725 T11 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33816/solr
   [junit4]   2> 15796 T11 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 15797 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 15799 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 15804 T35 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fd47b1e name:ZooKeeperConnection Watcher:127.0.0.1:33816 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 15805 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 15806 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 15839 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 15842 T38 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ec66173 name:ZooKeeperConnection Watcher:127.0.0.1:33816/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 15842 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 15869 T11 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 15892 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 15904 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 15909 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 15914 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 15942 T11 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 15945 T11 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42702_
   [junit4]   2> 15957 T11 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42702_
   [junit4]   2> 15962 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 15965 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 15968 T11 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 15997 T11 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42702_
   [junit4]   2> 15998 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 16001 T11 oasc.Overseer.start Overseer (id=93250541395902467-127.0.0.1:42702_-n_0000000000) starting
   [junit4]   2> 16028 T11 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 16163 T11 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 16842 T40 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 16843 T11 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 16848 T11 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 16850 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 17006 T39 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 17411 T11 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores
   [junit4]   2> 17708 T11 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores/collection1/
   [junit4]   2> 17709 T11 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 17720 T42 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 17721 T42 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 17747 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 17747 T42 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 17763 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:42702_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42702",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 17854 T39 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:42702_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42702",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 17867 T39 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 17943 T39 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 17972 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 18750 T42 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 18750 T42 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 18752 T42 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 18753 T42 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 18755 T42 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 18763 T42 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores/collection1/'
   [junit4]   2> 18838 T42 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 19003 T42 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 19095 T42 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 19260 T42 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 19558 T42 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 19601 T42 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 19754 T42 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 22120 T42 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 22201 T42 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 22218 T42 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 24007 T42 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 24059 T42 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 24069 T42 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 24081 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 24082 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 24083 T42 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 24085 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 24085 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 24120 T42 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 24544 T42 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 24590 T42 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/control/data/
   [junit4]   2> 24610 T42 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc96741
   [junit4]   2> 25041 T42 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/control/data
   [junit4]   2> 25042 T42 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/control/data/index/
   [junit4]   2> 25043 T42 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 25088 T42 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/control/data/index
   [junit4]   2> 25092 T42 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=0.7529296875, floorSegmentMB=1.0263671875, forceMergeDeletesPctAllowed=0.06681030112908437, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.26137159185459835
   [junit4]   2> 25624 T42 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53268e44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8463d02),segFN=segments_1,generation=1}
   [junit4]   2> 25635 T42 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 27385 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 27580 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 27611 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 27612 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 27612 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 27613 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 27792 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 27793 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 27793 T42 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 27924 T42 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 27926 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 27928 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 27929 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 27932 T42 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 27934 T42 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 27973 T42 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 27983 T42 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 27986 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 27989 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 27991 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 27993 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 27994 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 27996 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 27998 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 28000 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 28033 T42 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 28076 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 28374 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28423 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28426 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28428 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 28440 T42 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 28632 T42 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 28633 T42 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 28636 T42 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=76.71875, floorSegmentMB=2.1845703125, forceMergeDeletesPctAllowed=18.306141207435715, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 28652 T42 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53268e44 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8463d02),segFN=segments_1,generation=1}
   [junit4]   2> 28652 T42 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 28797 T42 oass.SolrIndexSearcher.<init> Opening Searcher@4b1ee15a[collection1] main
   [junit4]   2> 28857 T42 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 28859 T42 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 28860 T42 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 28878 T42 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 28879 T42 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 29019 T42 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 29021 T42 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 29022 T42 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 29023 T42 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 29305 T42 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 29320 T43 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b1ee15a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 29330 T42 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 29340 T46 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42702 collection:control_collection shard:shard1
   [junit4]   2> 29341 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 29342 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 29392 T46 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 29403 T46 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 29407 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 29408 T46 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 29408 T46 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 29409 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 29427 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@b73e739 url=http://127.0.0.1:42702/collection1 node=127.0.0.1:42702_ C0_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:42702_, core=collection1, base_url=http://127.0.0.1:42702, state=down}
   [junit4]   2> 29436 T46 C0 P42702 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42702/collection1/
   [junit4]   2> 29446 T46 C0 P42702 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 29447 T46 C0 P42702 oasc.SyncStrategy.syncToMe http://127.0.0.1:42702/collection1/ has no replicas
   [junit4]   2> 29447 T46 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42702/collection1/ shard1
   [junit4]   2> 29471 T46 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 29483 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 29486 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42702",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 29529 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 29531 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29533 T51 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5177bcb name:ZooKeeperConnection Watcher:127.0.0.1:33816/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29534 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29535 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 29537 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 29547 T11 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 29548 T11 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 29548 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 29550 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29552 T54 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ee13315 name:ZooKeeperConnection Watcher:127.0.0.1:33816/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29553 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29554 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 29558 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 29561 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 29561 T39 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 29564 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 29674 T39 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 29677 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 29739 T46 oasc.ZkController.register We are http://127.0.0.1:42702/collection1/ and leader is http://127.0.0.1:42702/collection1/
   [junit4]   2> 29750 T46 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42702
   [junit4]   2> 29750 T46 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 29751 T46 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 29751 T46 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 29754 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 29759 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:42702_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42702",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 29760 T39 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:42702_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42702",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 29868 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 29868 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 31623 T11 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores/collection1
   [junit4]   2> 31628 T11 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 31629 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 31633 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23945
   [junit4]   2> 31634 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 31635 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 31635 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001
   [junit4]   2> 31636 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/'
   [junit4]   2> 31702 T11 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/solr.xml
   [junit4]   2> 31813 T11 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores
   [junit4]   2> 31814 T11 oasc.CoreContainer.<init> New CoreContainer 1230744501
   [junit4]   2> 31815 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/]
   [junit4]   2> 31819 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 31820 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 31821 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 31822 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 31823 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 31824 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 31825 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 31826 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 31827 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 31828 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 31829 T11 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 31831 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 31832 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 31833 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 31834 T11 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 31835 T11 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33816/solr
   [junit4]   2> 31836 T11 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 31837 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 31838 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 31842 T66 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f117668 name:ZooKeeperConnection Watcher:127.0.0.1:33816 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 31843 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 31845 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 31848 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 31851 T69 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21ffd3d7 name:ZooKeeperConnection Watcher:127.0.0.1:33816/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 31852 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 31863 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 32875 T11 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38463_
   [junit4]   2> 32877 T11 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38463_
   [junit4]   2> 32884 T11 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 32899 T11 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores
   [junit4]   2> 32903 T11 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores/collection1/
   [junit4]   2> 32904 T11 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 32910 T70 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 32911 T70 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 32915 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 32917 T70 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 32917 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:38463_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38463",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 32917 T70 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 32919 T39 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:38463_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38463",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 32919 T70 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 32920 T39 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 32921 T39 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 32925 T39 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 32933 T70 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 32934 T70 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 32934 T70 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 32935 T70 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 32936 T70 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 32937 T70 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 32939 T70 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 32939 T70 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 32967 T70 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 32973 T70 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 33008 T70 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33073 T70 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 33207 T70 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33210 T70 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 33342 T70 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33983 T70 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34007 T70 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34011 T70 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34064 T70 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34074 T70 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34084 T70 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34086 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34087 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34088 T70 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34090 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34090 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34091 T70 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 34092 T70 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34092 T70 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/jetty1/
   [junit4]   2> 34093 T70 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1dc96741
   [junit4]   2> 34096 T70 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/jetty1
   [junit4]   2> 34097 T70 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/jetty1/index/
   [junit4]   2> 34097 T70 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 34098 T70 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1EB0D3AF9780128D-001/tempDir-001/jetty1/index
   [junit4]   2> 34099 T70 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=0.7529296875, floorSegmentMB=1.0263671875, forceMergeDeletesPctAllowed=0.06681030112908437, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.26137159185459835
   [junit4]   2> 34101 T70 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5ff0cbb9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@39cf0fe4),segFN=segments_1,generation=1}
   [junit4]   2> 34101 T70 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34118 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 34119 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34119 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34120 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34121 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34121 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 34122 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34123 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34124 T70 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 34128 T70 oasc.RequestHandlers.initHandlersFromConfig created /u

[...truncated too long message...]

tionTest -Dtests.method=test -Dtests.seed=1EB0D3AF9780128D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=nl -Dtests.timezone=America/Sao_Paulo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    144s J3 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53167/u_mst/df/c8n_1x3_lf_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1EB0D3AF9780128D:96E4EC75397C7F75]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53167/u_mst/df/c8n_1x3_lf_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 54 more
   [junit4]   2> 2020848 T4154 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderFailoverAfterPartitionTest 1EB0D3AF9780128D-001
   [junit4]   2> 143617 T4153 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=BlockTreeOrds(blocksize=128), multiDefault=BlockTreeOrds(blocksize=128), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), a_t=BlockTreeOrds(blocksize=128), _version_=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Asserting), text=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=nl, timezone=America/Sao_Paulo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=209889224,total=438304768
   [junit4]   2> NOTE: All tests run in this JVM: [LeaderInitiatedRecoveryOnCommitTest, SoftAutoCommitTest, BadIndexSchemaTest, TestCloudPivotFacet, ReplicationFactorTest, TestSystemIdResolver, HighlighterMaxOffsetTest, ZkCLITest, SolrCoreTest, XsltUpdateRequestHandlerTest, DocValuesMissingTest, TestImplicitCoreProperties, TestSchemaVersionResource, SharedFSAutoReplicaFailoverTest, OverriddenZkACLAndCredentialsProvidersTest, TestQuerySenderListener, TestTolerantSearch, TestRangeQuery, JsonLoaderTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestSolrConfigHandlerConcurrent, IndexSchemaRuntimeFieldTest, TestDocBasedVersionConstraints, TestCodecSupport, TestOverriddenPrefixQueryForCustomFieldType, DocValuesMultiTest, OutputWriterTest, MultiTermTest, BinaryUpdateRequestHandlerTest, TestJmxIntegration, TestDynamicFieldCollectionResource, SampleTest, SuggesterWFSTTest, TestReplicaProperties, CursorMarkTest, ResponseLogComponentTest, SpellPossibilityIteratorTest, FileUtilsTest, SpellingQueryConverterTest, CoreAdminHandlerTest, BlockDirectoryTest, DateMathParserTest, TestCSVLoader, TestStandardQParsers, TestSimpleQParserPlugin, TestGroupingSearch, TestWriterPerf, LeaderElectionTest, TestReRankQParserPlugin, TestMiniSolrCloudClusterSSL, ShardRoutingTest, DefaultValueUpdateProcessorTest, TestOrdValues, TestCloudInspectUtil, HdfsLockFactoryTest, TestFieldResource, UnloadDistributedZkTest, OpenCloseCoreStressTest, ClusterStateUpdateTest, TestZkChroot, TestRandomDVFaceting, DistributedTermsComponentTest, TestSolr4Spatial, TestFiltering, DirectUpdateHandlerTest, CurrencyFieldXmlFileTest, TestCoreDiscovery, TestExtendedDismaxParser, SuggesterFSTTest, DocValuesTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, TestSolrDeletionPolicy1, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, TestCollationField, ReturnFieldsTest, QueryParsingTest, TestPHPSerializedResponseWriter, TestComponentsName, HighlighterConfigTest, AlternateDirectoryTest, TestDocSet, TestFuzzyAnalyzedSuggestions, TestPhraseSuggestions, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, TimeZoneUtilsTest, ChaosMonkeyNothingIsSafeTest, DeleteShardTest, DistributedQueueTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J3 in 144.40s, 1 test, 1 error <<< FAILURES!

[...truncated 299 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1848 tests, 5 errors, 56 ignored (9 assumptions)

Total time: 147 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 699584548 bytes
Compression is 0.0%
Took 2 min 9 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 750 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/750/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.CloudExitableDirectoryReaderTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:32267/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:32267/collection1]
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:349)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1009)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1384)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.assertPartialResults(CloudExitableDirectoryReaderTest.java:102)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.doTimeoutTests(CloudExitableDirectoryReaderTest.java:86)
	at org.apache.solr.cloud.CloudExitableDirectoryReaderTest.test(CloudExitableDirectoryReaderTest.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:32267/collection1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	... 51 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46635/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46635/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9612 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/init-core-data-001
   [junit4]   2> 134441 T1147 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 134447 T1147 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 134448 T1148 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 134548 T1147 oasc.ZkTestServer.run start zk server on port:46584
   [junit4]   2> 134549 T1147 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 134550 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134555 T1155 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31902140 name:ZooKeeperConnection Watcher:127.0.0.1:46584 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134556 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134557 T1147 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 134557 T1147 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 134561 T1147 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 134562 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134564 T1158 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@431d1c3 name:ZooKeeperConnection Watcher:127.0.0.1:46584/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134564 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134565 T1147 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 134565 T1147 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 134568 T1147 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 134570 T1147 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 134571 T1147 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 134573 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 134574 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 134577 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 134578 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 134680 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 134681 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 134683 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 134684 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 134686 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 134687 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 134689 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 134689 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 134691 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 134692 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 134694 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 134695 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 134697 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 134698 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 134700 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 134701 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 134703 T1147 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 134703 T1147 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 136087 T1147 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores/collection1
   [junit4]   2> 136092 T1147 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 136096 T1147 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:46606
   [junit4]   2> 136097 T1147 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 136097 T1147 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 136098 T1147 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001
   [junit4]   2> 136098 T1147 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/'
   [junit4]   2> 136152 T1147 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/solr.xml
   [junit4]   2> 136212 T1147 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores
   [junit4]   2> 136213 T1147 oasc.CoreContainer.<init> New CoreContainer 565452929
   [junit4]   2> 136213 T1147 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/]
   [junit4]   2> 136215 T1147 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 136215 T1147 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 136215 T1147 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 136216 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 136216 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 136217 T1147 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 136217 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 136217 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 136218 T1147 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 136218 T1147 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 136218 T1147 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 136219 T1147 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 136219 T1147 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 136220 T1147 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 136220 T1147 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 136221 T1147 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46584/solr
   [junit4]   2> 136221 T1147 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 136221 T1147 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136222 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136224 T1170 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f42cc2c name:ZooKeeperConnection Watcher:127.0.0.1:46584 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136225 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136226 T1147 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136229 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136231 T1173 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2657d257 name:ZooKeeperConnection Watcher:127.0.0.1:46584/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136231 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136234 T1147 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 136237 T1147 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 136239 T1147 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 136242 T1147 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 136244 T1147 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 136246 T1147 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 136248 T1147 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46605_
   [junit4]   2> 136248 T1147 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46605_
   [junit4]   2> 136251 T1147 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 136252 T1147 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 136254 T1147 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 136256 T1147 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:46605_
   [junit4]   2> 136256 T1147 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 136258 T1147 oasc.Overseer.start Overseer (id=93248253674323971-127.0.0.1:46605_-n_0000000000) starting
   [junit4]   2> 136260 T1147 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 136267 T1147 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 136268 T1175 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 136269 T1147 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 136271 T1147 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 136272 T1147 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 136275 T1174 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 136276 T1147 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores
   [junit4]   2> 136279 T1147 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores/collection1/
   [junit4]   2> 136279 T1147 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 136281 T1177 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 136281 T1177 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 136282 T1173 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136282 T1177 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 136285 T1174 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46605_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46605",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 136286 T1174 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46605_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46605",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 136286 T1174 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 136286 T1174 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 136288 T1173 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 137283 T1177 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 137284 T1177 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 137284 T1177 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 137285 T1177 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 137285 T1177 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 137286 T1177 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores/collection1/'
   [junit4]   2> 137304 T1177 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 137307 T1177 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 137320 T1177 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 137340 T1177 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 137438 T1177 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 137539 T1177 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 137549 T1177 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 137872 T1177 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 137882 T1177 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 137885 T1177 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 137897 T1177 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137902 T1177 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137905 T1177 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 137906 T1177 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 137907 T1177 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 137907 T1177 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 137908 T1177 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 137908 T1177 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 137908 T1177 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 137908 T1177 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 137908 T1177 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/tempDir-001/control/data/
   [junit4]   2> 137909 T1177 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16dace85
   [junit4]   2> 137909 T1177 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/tempDir-001/control/data
   [junit4]   2> 137910 T1177 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/tempDir-001/control/data/index/
   [junit4]   2> 137910 T1177 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 137910 T1177 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/tempDir-001/control/data/index
   [junit4]   2> 137910 T1177 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8720928946083775]
   [junit4]   2> 137911 T1177 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47d1075d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6c3a8f80),segFN=segments_1,generation=1}
   [junit4]   2> 137911 T1177 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137916 T1177 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 137916 T1177 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 137916 T1177 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 137916 T1177 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137917 T1177 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137917 T1177 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 137917 T1177 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137917 T1177 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137917 T1177 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 137918 T1177 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 137918 T1177 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 137919 T1177 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 137919 T1177 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 137920 T1177 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 137920 T1177 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 137920 T1177 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 137921 T1177 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 137921 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 137922 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 137923 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 137923 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 137924 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 137924 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 137924 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 137925 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 137925 T1177 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 137926 T1177 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 137945 T1177 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137946 T1177 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137948 T1177 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137950 T1177 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137951 T1177 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 137955 T1177 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 137955 T1177 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 137956 T1177 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=47.98046875, floorSegmentMB=0.568359375, forceMergeDeletesPctAllowed=7.296329178022043, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 137956 T1177 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47d1075d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6c3a8f80),segFN=segments_1,generation=1}
   [junit4]   2> 137956 T1177 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137957 T1177 oass.SolrIndexSearcher.<init> Opening Searcher@5a873a1[collection1] main
   [junit4]   2> 137957 T1177 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 137958 T1177 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 137958 T1177 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 137958 T1177 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 137959 T1177 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 137959 T1177 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 137959 T1177 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 137960 T1177 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 137960 T1177 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 137960 T1177 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 137961 T1178 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5a873a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 137961 T1177 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 137962 T1181 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:46605 collection:control_collection shard:shard1
   [junit4]   2> 137962 T1147 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 137963 T1147 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 137963 T1181 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 137967 T1147 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 137967 T1181 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 137968 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137969 T1173 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 137969 T1181 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 137969 T1185 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@603b1acd name:ZooKeeperConnection Watcher:127.0.0.1:46584/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137970 T1174 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 137970 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137969 T1181 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C99 name=collection1 org.apache.solr.core.SolrCore@168812f url=http://127.0.0.1:46605/collection1 node=127.0.0.1:46605_ C99_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:46605_, base_url=http://127.0.0.1:46605, core=collection1, state=down}
   [junit4]   2> 137970 T1181 C99 P46605 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:46605/collection1/
   [junit4]   2> 137971 T1147 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 137971 T1173 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 137971 T1181 C99 P46605 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 137972 T1181 C99 P46605 oasc.SyncStrategy.syncToMe http://127.0.0.1:46605/collection1/ has no replicas
   [junit4]   2> 137972 T1147 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 137972 T1181 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:46605/collection1/ shard1
   [junit4]   2> 137973 T1181 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 137975 T1147 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 137976 T1186 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 137977 T1174 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46605",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 138080 T1185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138129 T1181 oasc.ZkController.register We are http://127.0.0.1:46605/collection1/ and leader is http://127.0.0.1:46605/collection1/
   [junit4]   2> 138130 T1181 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46605
   [junit4]   2> 138130 T1181 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 138130 T1181 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 138130 T1181 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 138132 T1173 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 138133 T1174 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46605_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46605",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 138134 T1174 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46605_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46605",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 138237 T1173 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138237 T1185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139415 T1147 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/cores/collection1
   [junit4]   2> 139418 T1147 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 139419 T1147 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 139422 T1147 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:46636
   [junit4]   2> 139424 T1147 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 139424 T1147 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 139425 T1147 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001
   [junit4]   2> 139425 T1147 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/'
   [junit4]   2> 139470 T1147 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/solr.xml
   [junit4]   2> 139524 T1147 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/cores
   [junit4]   2> 139525 T1147 oasc.CoreContainer.<init> New CoreContainer 668492798
   [junit4]   2> 139525 T1147 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/]
   [junit4]   2> 139526 T1147 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 139527 T1147 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 139527 T1147 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 139528 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 139528 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 139528 T1147 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 139529 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 139529 T1147 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 139529 T1147 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 139530 T1147 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 139530 T1147 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 139531 T1147 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 139531 T1147 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 139532 T1147 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 139532 T1147 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 139532 T1147 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46584/solr
   [junit4]   2> 139533 T1147 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 139533 T1147 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 139534 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 139536 T1198 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68f60dc8 name:ZooKeeperConnection Watcher:127.0.0.1:46584 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 139537 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 139538 T1147 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 139541 T1147 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 139542 T1201 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69e9b95 name:ZooKeeperConnection Watcher:127.0.0.1:46584/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 139543 T1147 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 139549 T1147 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 140552 T1147 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46635_
   [junit4]   2> 140553 T1147 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46635_
   [junit4]   2> 140557 T1147 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 140560 T1147 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/cores
   [junit4]   2> 140563 T1147 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/cores/collection1/
   [junit4]   2> 140563 T1147 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 140564 T1202 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 140565 T1202 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 140566 T1202 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 140566 T1173 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 140567 T1174 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46635_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46635",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 140567 T1174 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46635_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46635",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 140568 T1174 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 140568 T1174 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 140569 T1173 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 140570 T1201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 140570 T1185 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 141566 T1202 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 141567 T1202 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 141568 T1202 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 141568 T1202 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 141569 T1202 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 141569 T1202 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 38860A5A3288A3C-001/shard-1-001/cores/collection1/'
   [junit4]   2> 141586 T1202 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 141589 T1202 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 141601 T1202 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 141625 T1202 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 141677 T1202 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 141778 T1202 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 141790 T1202 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 142173 T1202 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 142185 T1202 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 142188 T1202 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 142205 T1202 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 142210 T1202 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 142214 T1202 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 142215 T1202 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 142216 T1202 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 142217 T1202 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 142217 T1202 oass.OpenExchangeRatesOrgProvider$O

[...truncated too long message...]

:60262/
   [junit4]   2> 1966658 T7174 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1966658 T7174 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:60249/
   [junit4]   2> 1966659 T7174 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:60229/
   [junit4]   2> 1966659 T7174 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:60272/
   [junit4]   2> 1966659 T7174 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:60262/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=38860A5A3288A3C -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_DZ -Dtests.timezone=America/Costa_Rica -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   68.7s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([38860A5A3288A3C:8BDC5F7F0DD4E7C4]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1966671 T7174 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 38860A5A3288A3C-001
   [junit4]   2> 68710 T7173 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=SimpleText, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_DZ, timezone=America/Costa_Rica
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=119856400,total=353370112
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsLockFactoryTest, SharedFSAutoReplicaFailoverTest, ClusterStateTest, SampleTest, CurrencyFieldXmlFileTest, TestDistributedGrouping, SuggesterWFSTTest, TestJmxMonitoredMap, ExternalCollectionsTest, AlternateDirectoryTest, BinaryUpdateRequestHandlerTest, TestCloudInspectUtil, LeaderInitiatedRecoveryOnCommitTest, DocValuesMultiTest, TestDefaultSimilarityFactory, DocValuesMissingTest, XsltUpdateRequestHandlerTest, TestJoin, ZkCLITest, FileUtilsTest, TestSolrConfigHandlerCloud, TestDynamicFieldCollectionResource, TestQuerySenderListener, ScriptEngineTest, RemoteQueryErrorTest, TestMissingGroups, TestFiltering, TestSurroundQueryParser, TestConfigOverlay, TestArbitraryIndexDir, TestIndexSearcher, TestSchemaVersionResource, ClusterStateUpdateTest, TestFieldResource, ResponseLogComponentTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestValueSourceCache, TestQueryTypes, TestBulkSchemaConcurrent, AnalysisAfterCoreReloadTest, TestBlendedInfixSuggestions, TestRangeQuery, CursorMarkTest, NumericFieldsTest, ShardRoutingTest, SolrCoreTest, TestWriterPerf, BadIndexSchemaTest, TestRequestStatusCollectionAPI, CSVRequestHandlerTest, TestCSVLoader, TestImplicitCoreProperties, TestReRankQParserPlugin, BlockDirectoryTest, HdfsDirectoryFactoryTest, DeleteLastCustomShardedReplicaTest, BasicDistributedZk2Test, SyncSliceTest, OverseerTest, BasicZkTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, ZkSolrClientTest, TestZkChroot, AutoCommitTest, DistributedSpellCheckComponentTest, ZkControllerTest, DistributedTermsComponentTest, StatsComponentTest, DirectUpdateHandlerTest, HighlighterTest, SimplePostToolTest, SuggesterFSTTest, CoreAdminHandlerTest, TestStressLucene, SpatialFilterTest, TestUpdate, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, StandardRequestHandlerTest, FileBasedSpellCheckerTest, TestIndexingPerformance, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCollationField, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, TestBinaryResponseWriter, SOLR749Test, CopyFieldTest, MinimalSchemaTest, OutputWriterTest, ExternalFileFieldSortTest, TestLuceneMatchVersion, TestCodecSupport, TestLMDirichletSimilarityFactory, ResourceLoaderTest, URLClassifyProcessorTest, SpellingQueryConverterTest, TestUtils, TestTolerantSearch, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, DeleteShardTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 69.49s, 1 test, 1 error <<< FAILURES!

[...truncated 323 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1848 tests, 6 errors, 56 ignored (9 assumptions)

Total time: 152 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 726753347 bytes
Compression is 0.0%
Took 2 min 0 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 749 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/749/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=4624, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4624, name=collection5, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51742/nfj/q: collection already exists: awholynewstresscollection_collection5_5
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1009)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1631)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1652)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:879)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32007/nd_lfp/wa/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32007/nd_lfp/wa/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10063 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/init-core-data-001
   [junit4]   2> 346879 T440 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 346887 T440 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 346888 T441 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 346987 T440 oasc.ZkTestServer.run start zk server on port:33610
   [junit4]   2> 346988 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 346989 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 346993 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@768d4247 name:ZooKeeperConnection Watcher:127.0.0.1:33610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 346994 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 346994 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 346995 T440 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 346998 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 346999 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347001 T451 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ae9dcf name:ZooKeeperConnection Watcher:127.0.0.1:33610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347001 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347002 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 347002 T440 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 347004 T440 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 347006 T440 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 347007 T440 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 347009 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 347010 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 347013 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 347014 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 347116 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 347117 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 347119 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 347119 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 347121 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 347122 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 347124 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 347124 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 347126 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 347127 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 347128 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 347129 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 347131 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 347131 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 347133 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 347134 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 347135 T440 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 347136 T440 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 348630 T440 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores/collection1
   [junit4]   2> 348633 T440 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 348637 T440 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33616
   [junit4]   2> 348638 T440 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 348638 T440 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 348638 T440 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001
   [junit4]   2> 348638 T440 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/'
   [junit4]   2> 348673 T440 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/solr.xml
   [junit4]   2> 348721 T440 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores
   [junit4]   2> 348722 T440 oasc.CoreContainer.<init> New CoreContainer 536715588
   [junit4]   2> 348722 T440 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/]
   [junit4]   2> 348724 T440 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 348724 T440 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 348725 T440 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 348725 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 348725 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 348726 T440 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 348726 T440 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 348726 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 348727 T440 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 348727 T440 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 348727 T440 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 348728 T440 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 348729 T440 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 348729 T440 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 348729 T440 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 348730 T440 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33610/solr
   [junit4]   2> 348730 T440 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 348731 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 348732 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348733 T463 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@274ddd7 name:ZooKeeperConnection Watcher:127.0.0.1:33610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348734 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348735 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 348737 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348739 T466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@567e779 name:ZooKeeperConnection Watcher:127.0.0.1:33610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348739 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348742 T440 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 348744 T440 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 348746 T440 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 348748 T440 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 348750 T440 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 348752 T440 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 348753 T440 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33615_
   [junit4]   2> 348754 T440 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33615_
   [junit4]   2> 348755 T440 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 348757 T440 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 348758 T440 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 348759 T440 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33615_
   [junit4]   2> 348759 T440 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 348760 T440 oasc.Overseer.start Overseer (id=93245023103811587-127.0.0.1:33615_-n_0000000000) starting
   [junit4]   2> 348762 T440 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 348766 T440 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 348767 T468 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 348767 T440 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 348769 T440 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 348770 T440 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 348772 T467 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 348773 T440 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores
   [junit4]   2> 348775 T440 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores/collection1/
   [junit4]   2> 348775 T440 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 348776 T470 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 348777 T470 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 348778 T470 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 348778 T466 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 348780 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33615",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33615_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 348780 T467 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33615",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33615_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 348780 T467 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 348781 T467 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 348782 T466 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 349778 T470 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 349779 T470 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 349779 T470 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 349780 T470 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 349780 T470 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 349781 T470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores/collection1/'
   [junit4]   2> 349799 T470 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 349801 T470 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 349815 T470 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 349839 T470 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 349892 T470 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 349994 T470 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 350004 T470 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 350291 T470 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 350300 T470 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 350302 T470 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 350314 T470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 350318 T470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 350321 T470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 350321 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 350322 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 350322 T470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 350323 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 350323 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 350323 T470 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 350323 T470 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 350324 T470 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/control/data/
   [junit4]   2> 350324 T470 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@222c9a95
   [junit4]   2> 350324 T470 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/control/data
   [junit4]   2> 350325 T470 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/control/data/index/
   [junit4]   2> 350325 T470 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 350326 T470 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/control/data/index
   [junit4]   2> 350326 T470 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=989905165, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 350330 T470 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/index-NIOFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@56d97688),segFN=segments_1,generation=1}
   [junit4]   2> 350330 T470 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 350335 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 350335 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 350335 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 350336 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 350336 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 350336 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 350336 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 350337 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 350337 T470 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 350337 T470 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 350338 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 350338 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 350338 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 350339 T470 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 350339 T470 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 350339 T470 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 350340 T470 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 350340 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 350341 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 350341 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 350342 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 350342 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 350343 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 350343 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 350343 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 350344 T470 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 350344 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 350361 T470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 350363 T470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 350365 T470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 350367 T470 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 350369 T470 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 350373 T470 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 350373 T470 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 350374 T470 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=748440242, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 350377 T470 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/index-NIOFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@56d97688),segFN=segments_1,generation=1}
   [junit4]   2> 350377 T470 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 350378 T470 oass.SolrIndexSearcher.<init> Opening Searcher@151e9a09[collection1] main
   [junit4]   2> 350378 T470 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 350379 T470 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 350379 T470 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 350380 T470 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 350380 T470 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 350380 T470 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 350381 T470 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 350381 T470 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 350381 T470 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 350382 T470 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 350383 T471 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@151e9a09[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 350383 T470 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 350384 T474 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33615 collection:control_collection shard:shard1
   [junit4]   2> 350385 T440 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 350385 T440 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 350385 T474 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 350388 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 350389 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 350390 T474 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 350391 T478 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13e213d8 name:ZooKeeperConnection Watcher:127.0.0.1:33610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 350391 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 350392 T466 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 350392 T474 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 350392 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 350392 T474 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 350393 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C174 name=collection1 org.apache.solr.core.SolrCore@72888249 url=http://127.0.0.1:33615/collection1 node=127.0.0.1:33615_ C174_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:33615, state=down, node_name=127.0.0.1:33615_, core=collection1}
   [junit4]   2> 350393 T474 C174 P33615 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33615/collection1/
   [junit4]   2> 350393 T440 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 350394 T474 C174 P33615 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 350394 T474 C174 P33615 oasc.SyncStrategy.syncToMe http://127.0.0.1:33615/collection1/ has no replicas
   [junit4]   2> 350394 T474 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33615/collection1/ shard1
   [junit4]   2> 350395 T474 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 350395 T478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 350395 T466 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350397 T440 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 350397 T440 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 350398 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 350398 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 350399 T481 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 350400 T482 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49b2d6d8 name:ZooKeeperConnection Watcher:127.0.0.1:33610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 350400 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33615",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 350401 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 350401 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 350404 T481 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 350405 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 350405 T467 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 350406 T478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350510 T467 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 350512 T478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350552 T474 oasc.ZkController.register We are http://127.0.0.1:33615/collection1/ and leader is http://127.0.0.1:33615/collection1/
   [junit4]   2> 350553 T474 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33615
   [junit4]   2> 350553 T474 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 350553 T474 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 350553 T474 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 350555 T466 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 350557 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33615",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33615_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 4
   [junit4]   2> 350557 T467 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33615",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33615_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 350661 T466 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350661 T478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351908 T440 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores/collection1
   [junit4]   2> 351911 T440 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 351911 T440 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 351914 T440 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33631
   [junit4]   2> 351914 T440 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 351915 T440 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 351915 T440 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001
   [junit4]   2> 351915 T440 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/'
   [junit4]   2> 351950 T440 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/solr.xml
   [junit4]   2> 351991 T440 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores
   [junit4]   2> 351991 T440 oasc.CoreContainer.<init> New CoreContainer 138391789
   [junit4]   2> 351991 T440 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/]
   [junit4]   2> 351992 T440 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 351993 T440 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 351993 T440 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 351993 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 351993 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 351994 T440 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 351994 T440 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 351994 T440 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 351994 T440 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 351994 T440 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 351995 T440 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 351995 T440 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 351995 T440 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 351996 T440 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 351996 T440 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 351996 T440 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33610/solr
   [junit4]   2> 351996 T440 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 351997 T440 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 351997 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 351999 T494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b659b3a name:ZooKeeperConnection Watcher:127.0.0.1:33610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352000 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352001 T440 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 352004 T440 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352005 T497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25d0a2a5 name:ZooKeeperConnection Watcher:127.0.0.1:33610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352005 T440 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352010 T440 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 353015 T440 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33630_
   [junit4]   2> 353016 T440 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33630_
   [junit4]   2> 353019 T440 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 353022 T440 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores
   [junit4]   2> 353024 T440 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores/collection1/
   [junit4]   2> 353025 T440 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 353027 T498 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 353027 T498 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 353028 T466 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 353028 T498 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 353028 T498 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 353029 T467 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33630",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33630_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 353029 T498 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 353029 T467 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33630",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33630_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 353029 T467 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 353030 T498 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 353030 T467 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 353030 T498 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 353031 T467 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 353032 T497 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 353032 T497 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 354030 T498 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 354031 T498 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 354031 T498 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 354032 T498 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 354032 T498 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 354033 T498 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 354051 T498 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 354053 T498 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 354067 T498 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 354092 T498 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 354146 T498 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 354248 T498 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 354263 T498 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 354675 T498 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 354689 T498 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 354692 T498 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 354706 T498 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 354711 T498 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 354716 T498 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 354717 T498 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 354718 T498 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 354718 T498 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 354719 T498 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 354719 T498 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 354720 T498 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 354720 T498 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 354720 T498 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/jetty1/
   [junit4]   2> 354720 T498 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@222c9a95
   [junit4]   2> 354721 T498 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/jetty1
   [junit4]   2> 354722 T498 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/jetty1/index/
   [junit4]   2> 354722 T498 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 354725 T498 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 89351B3CD1E4CDAD-001/tempDir-001/jetty1/index
   [junit4]   2> 354725 T498 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=989905165, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 354732 T498 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=BaseDirectoryWrapper(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/wor

[...truncated too long message...]

4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2134043 T11571 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:35352/
   [junit4]   2> 2134043 T11571 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35369/
   [junit4]   2> 2134043 T11571 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:35331/
   [junit4]   2> 2134044 T11571 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:35342/
   [junit4]   2> 2134044 T11571 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2134044 T11571 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35352/
   [junit4]   2> 2134045 T11571 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35369/
   [junit4]   2> 2134045 T11571 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35331/
   [junit4]   2> 2134045 T11571 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35342/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=89351B3CD1E4CDAD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=th_TH -Dtests.timezone=Etc/GMT-10 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    118s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([89351B3CD1E4CDAD:16124E67F18A055]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2134067 T11571 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 89351B3CD1E4CDAD-001
   [junit4]   2> 118341 T11570 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=15438, maxDocsPerChunk=111, blockSize=185), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=15438, blockSize=185)), sim=DefaultSimilarity, locale=th_TH, timezone=Etc/GMT-10
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=84188160,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [QueryParsingTest, TestSchemaResource, DebugComponentTest, OverseerTest, MergeStrategyTest, VMParamsZkACLAndCredentialsProvidersTest, TestCoreContainer, TestFreeTextSuggestions, MBeansHandlerTest, CachingDirectoryFactoryTest, TestSolrConfigHandler, TestCollationField, TestDefaultStatsCache, HighlighterConfigTest, DistribCursorPagingTest, TestFastLRUCache, ZkSolrClientTest, RequiredFieldsTest, TestClassNameShortening, SignatureUpdateProcessorFactoryTest, PrimitiveFieldTypeTest, UUIDFieldTest, PolyFieldTest, TestMiniSolrCloudCluster, SimpleFacetsTest, TestDFRSimilarityFactory, DistributedFacetPivotWhiteBoxTest, IndexBasedSpellCheckerTest, TestManagedSchemaDynamicFieldResource, TestPivotHelperCode, TestManagedSynonymFilterFactory, MinimalSchemaTest, DOMUtilTest, TestSolrJ, ExternalFileFieldSortTest, TestDynamicLoading, TestCloudSchemaless, DistributedExpandComponentTest, HdfsBasicDistributedZkTest, TestSolrCoreProperties, AtomicUpdatesTest, DistributedQueryElevationComponentTest, TestHighlightDedupGrouping, TestChildDocTransformer, ShowFileRequestHandlerTest, TestIBSimilarityFactory, SaslZkACLProviderTest, DistributedDebugComponentTest, AnalyticsMergeStrategyTest, HdfsCollectionsAPIDistributedZkTest, TestTrie, TestStressReorder, TestUniqueKeyFieldResource, DistribDocExpirationUpdateProcessorTest, DirectSolrSpellCheckerTest, BasicDistributedZkTest, SyncSliceTest, LeaderElectionIntegrationTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, ZkCLITest, TestDistributedSearch, TestCursorMarkWithoutUniqueKey, AliasIntegrationTest, AssignTest, CloudExitableDirectoryReaderTest, DeleteInactiveReplicaTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 118.87s, 1 test, 1 error <<< FAILURES!

[...truncated 277 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1847 tests, 6 errors, 56 ignored (9 assumptions)

Total time: 339 minutes 18 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 666020814 bytes
Compression is 0.0%
Took 1 min 59 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 748 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/748/

6 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1422547211528,generation=2,filelist=[_8d.cfe, _8d.cfs, _8d.si, _8e.fdt, _8e.fdx, _8e.fnm, _8e.nvd, _8e.nvm, _8e.si, _8e_TestBloomFilteredLucenePostings_0.blm, _8e_TestBloomFilteredLucenePostings_0.doc, _8e_TestBloomFilteredLucenePostings_0.tim, _8e_TestBloomFilteredLucenePostings_0.tip, _8f.cfe, _8f.cfs, _8f.si, _8g.cfe, _8g.cfs, _8g.si, _8h.cfe, _8h.cfs, _8h.si, _8i.cfe, _8i.cfs, _8i.si, _8j.cfe, _8j.cfs, _8j.si, _8k.cfe, _8k.cfs, _8k.si, _8l.cfe, _8l.cfs, _8l.si, _8m.cfe, _8m.cfs, _8m.si, _8n.cfe, _8n.cfs, _8n.si, _8o.cfe, _8o.cfs, _8o.si, _8p.cfe, _8p.cfs, _8p.si, _8q.cfe, _8q.cfs, _8q.si, _8r.cfe, _8r.cfs, _8r.si, _8s.cfe, _8s.cfs, _8s.si, _8t.cfe, _8t.cfs, _8t.si, _8u.cfe, _8u.cfs, _8u.si, _8v.cfe, _8v.cfs, _8v.si, _8w.cfe, _8w.cfs, _8w.si, _8x.cfe, _8x.cfs, _8x.si, _8y.cfe, _8y.cfs, _8y.si, _8z.cfe, _8z.cfs, _8z.si, _90.cfe, _90.cfs, _90.si, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9m.cfe, _9m.cfs, _9m.si, segments_2]}]> but was:<[{indexVersion=1422547211528,generation=2,filelist=[_8d.cfe, _8d.cfs, _8d.si, _8e.fdt, _8e.fdx, _8e.fnm, _8e.nvd, _8e.nvm, _8e.si, _8e_TestBloomFilteredLucenePostings_0.blm, _8e_TestBloomFilteredLucenePostings_0.doc, _8e_TestBloomFilteredLucenePostings_0.tim, _8e_TestBloomFilteredLucenePostings_0.tip, _8f.cfe, _8f.cfs, _8f.si, _8g.cfe, _8g.cfs, _8g.si, _8h.cfe, _8h.cfs, _8h.si, _8i.cfe, _8i.cfs, _8i.si, _8j.cfe, _8j.cfs, _8j.si, _8k.cfe, _8k.cfs, _8k.si, _8l.cfe, _8l.cfs, _8l.si, _8m.cfe, _8m.cfs, _8m.si, _8n.cfe, _8n.cfs, _8n.si, _8o.cfe, _8o.cfs, _8o.si, _8p.cfe, _8p.cfs, _8p.si, _8q.cfe, _8q.cfs, _8q.si, _8r.cfe, _8r.cfs, _8r.si, _8s.cfe, _8s.cfs, _8s.si, _8t.cfe, _8t.cfs, _8t.si, _8u.cfe, _8u.cfs, _8u.si, _8v.cfe, _8v.cfs, _8v.si, _8w.cfe, _8w.cfs, _8w.si, _8x.cfe, _8x.cfs, _8x.si, _8y.cfe, _8y.cfs, _8y.si, _8z.cfe, _8z.cfs, _8z.si, _90.cfe, _90.cfs, _90.si, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9m.cfe, _9m.cfs, _9m.si, segments_2]}, {indexVersion=1422547211528,generation=3,filelist=[_9k.cfe, _9k.cfs, _9k.si, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_TestBloomFilteredLucenePostings_0.blm, _9l_TestBloomFilteredLucenePostings_0.doc, _9l_TestBloomFilteredLucenePostings_0.tim, _9l_TestBloomFilteredLucenePostings_0.tip, _9m.cfe, _9m.cfs, _9m.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1422547211528,generation=2,filelist=[_8d.cfe, _8d.cfs, _8d.si, _8e.fdt, _8e.fdx, _8e.fnm, _8e.nvd, _8e.nvm, _8e.si, _8e_TestBloomFilteredLucenePostings_0.blm, _8e_TestBloomFilteredLucenePostings_0.doc, _8e_TestBloomFilteredLucenePostings_0.tim, _8e_TestBloomFilteredLucenePostings_0.tip, _8f.cfe, _8f.cfs, _8f.si, _8g.cfe, _8g.cfs, _8g.si, _8h.cfe, _8h.cfs, _8h.si, _8i.cfe, _8i.cfs, _8i.si, _8j.cfe, _8j.cfs, _8j.si, _8k.cfe, _8k.cfs, _8k.si, _8l.cfe, _8l.cfs, _8l.si, _8m.cfe, _8m.cfs, _8m.si, _8n.cfe, _8n.cfs, _8n.si, _8o.cfe, _8o.cfs, _8o.si, _8p.cfe, _8p.cfs, _8p.si, _8q.cfe, _8q.cfs, _8q.si, _8r.cfe, _8r.cfs, _8r.si, _8s.cfe, _8s.cfs, _8s.si, _8t.cfe, _8t.cfs, _8t.si, _8u.cfe, _8u.cfs, _8u.si, _8v.cfe, _8v.cfs, _8v.si, _8w.cfe, _8w.cfs, _8w.si, _8x.cfe, _8x.cfs, _8x.si, _8y.cfe, _8y.cfs, _8y.si, _8z.cfe, _8z.cfs, _8z.si, _90.cfe, _90.cfs, _90.si, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9m.cfe, _9m.cfs, _9m.si, segments_2]}]> but was:<[{indexVersion=1422547211528,generation=2,filelist=[_8d.cfe, _8d.cfs, _8d.si, _8e.fdt, _8e.fdx, _8e.fnm, _8e.nvd, _8e.nvm, _8e.si, _8e_TestBloomFilteredLucenePostings_0.blm, _8e_TestBloomFilteredLucenePostings_0.doc, _8e_TestBloomFilteredLucenePostings_0.tim, _8e_TestBloomFilteredLucenePostings_0.tip, _8f.cfe, _8f.cfs, _8f.si, _8g.cfe, _8g.cfs, _8g.si, _8h.cfe, _8h.cfs, _8h.si, _8i.cfe, _8i.cfs, _8i.si, _8j.cfe, _8j.cfs, _8j.si, _8k.cfe, _8k.cfs, _8k.si, _8l.cfe, _8l.cfs, _8l.si, _8m.cfe, _8m.cfs, _8m.si, _8n.cfe, _8n.cfs, _8n.si, _8o.cfe, _8o.cfs, _8o.si, _8p.cfe, _8p.cfs, _8p.si, _8q.cfe, _8q.cfs, _8q.si, _8r.cfe, _8r.cfs, _8r.si, _8s.cfe, _8s.cfs, _8s.si, _8t.cfe, _8t.cfs, _8t.si, _8u.cfe, _8u.cfs, _8u.si, _8v.cfe, _8v.cfs, _8v.si, _8w.cfe, _8w.cfs, _8w.si, _8x.cfe, _8x.cfs, _8x.si, _8y.cfe, _8y.cfs, _8y.si, _8z.cfe, _8z.cfs, _8z.si, _90.cfe, _90.cfs, _90.si, _91.cfe, _91.cfs, _91.si, _92.cfe, _92.cfs, _92.si, _93.cfe, _93.cfs, _93.si, _94.cfe, _94.cfs, _94.si, _95.cfe, _95.cfs, _95.si, _96.cfe, _96.cfs, _96.si, _97.cfe, _97.cfs, _97.si, _98.cfe, _98.cfs, _98.si, _99.cfe, _99.cfs, _99.si, _9a.cfe, _9a.cfs, _9a.si, _9b.cfe, _9b.cfs, _9b.si, _9c.cfe, _9c.cfs, _9c.si, _9d.cfe, _9d.cfs, _9d.si, _9e.cfe, _9e.cfs, _9e.si, _9f.cfe, _9f.cfs, _9f.si, _9g.cfe, _9g.cfs, _9g.si, _9h.cfe, _9h.cfs, _9h.si, _9i.cfe, _9i.cfs, _9i.si, _9j.cfe, _9j.cfs, _9j.si, _9k.cfe, _9k.cfs, _9k.si, _9m.cfe, _9m.cfs, _9m.si, segments_2]}, {indexVersion=1422547211528,generation=3,filelist=[_9k.cfe, _9k.cfs, _9k.si, _9l.fdt, _9l.fdx, _9l.fnm, _9l.nvd, _9l.nvm, _9l.si, _9l_TestBloomFilteredLucenePostings_0.blm, _9l_TestBloomFilteredLucenePostings_0.doc, _9l_TestBloomFilteredLucenePostings_0.tim, _9l_TestBloomFilteredLucenePostings_0.tip, _9m.cfe, _9m.cfs, _9m.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:BF8FF9FF9363BC7E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56320/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56320/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:462)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56320/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52487/ma_nu/c8n_1x3_lf_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52487/ma_nu/c8n_1x3_lf_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:462)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52487/ma_nu/c8n_1x3_lf_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:23733/_nm/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23733/_nm/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23871/mf_eql/cg/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23871/mf_eql/cg/collection1
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23871/mf_eql/cg/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36996/cd_ulu/te/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36996/cd_ulu/te/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:295)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:209)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36996/cd_ulu/te/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 10066 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/init-core-data-001
   [junit4]   2> 1138416 T3037 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1138420 T3037 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1138421 T3038 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1138521 T3037 oasc.ZkTestServer.run start zk server on port:56301
   [junit4]   2> 1138522 T3037 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1138523 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1138526 T3045 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2677f1b2 name:ZooKeeperConnection Watcher:127.0.0.1:56301 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1138526 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1138526 T3037 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1138527 T3037 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1138529 T3037 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1138530 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1138531 T3048 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4547f2cc name:ZooKeeperConnection Watcher:127.0.0.1:56301/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1138531 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1138532 T3037 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1138532 T3037 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1138534 T3037 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1138536 T3037 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1138537 T3037 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1138538 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1138539 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1138542 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1138543 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1138645 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1138646 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1138648 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1138648 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1138650 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1138651 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1138652 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1138653 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1138655 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1138656 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1138658 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1138658 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1138660 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1138661 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1138663 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1138664 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1138666 T3037 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1138666 T3037 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1140785 T3037 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores/collection1
   [junit4]   2> 1140790 T3037 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1140793 T3037 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56321
   [junit4]   2> 1140794 T3037 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@1174ec5
   [junit4]   2> 1140794 T3037 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1140795 T3037 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001
   [junit4]   2> 1140795 T3037 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/'
   [junit4]   2> 1140846 T3037 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/solr.xml
   [junit4]   2> 1140898 T3037 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores
   [junit4]   2> 1140899 T3037 oasc.CoreContainer.<init> New CoreContainer 1786110287
   [junit4]   2> 1140899 T3037 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/]
   [junit4]   2> 1140900 T3037 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1140900 T3037 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1140901 T3037 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1140901 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1140902 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1140902 T3037 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1140902 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1140903 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1140903 T3037 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1140903 T3037 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1140904 T3037 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1140904 T3037 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1140905 T3037 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1140905 T3037 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1140905 T3037 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1140906 T3037 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56301/solr
   [junit4]   2> 1140906 T3037 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1140907 T3037 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1140907 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1140909 T3060 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58629a55 name:ZooKeeperConnection Watcher:127.0.0.1:56301 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1140910 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1140911 T3037 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1140913 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1140915 T3063 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36636172 name:ZooKeeperConnection Watcher:127.0.0.1:56301/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1140915 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1140918 T3037 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1140922 T3037 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1140925 T3037 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1140928 T3037 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1140931 T3037 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1140934 T3037 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1140936 T3037 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56320_
   [junit4]   2> 1140937 T3037 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56320_
   [junit4]   2> 1140939 T3037 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1140941 T3037 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1140943 T3037 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1140946 T3037 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:56320_
   [junit4]   2> 1140946 T3037 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1140949 T3037 oasc.Overseer.start Overseer (id=93227984670097411-127.0.0.1:56320_-n_0000000000) starting
   [junit4]   2> 1140952 T3037 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1140961 T3037 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1140962 T3065 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1140964 T3037 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1140967 T3037 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1140969 T3037 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1140974 T3037 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores
   [junit4]   2> 1140978 T3037 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores/collection1/
   [junit4]   2> 1140978 T3037 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1140997 T3064 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1140997 T3067 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1140998 T3067 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1140999 T3067 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1140999 T3063 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1141003 T3064 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56320",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56320_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1141004 T3064 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56320",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56320_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1141004 T3064 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1141006 T3064 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1141009 T3063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1142000 T3067 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1142001 T3067 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1142002 T3067 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1142002 T3067 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1142003 T3067 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1142004 T3067 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores/collection1/'
   [junit4]   2> 1142028 T3067 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1142031 T3067 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1142044 T3067 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1142071 T3067 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1142145 T3067 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1142246 T3067 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1142264 T3067 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1142845 T3067 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1142869 T3067 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1142875 T3067 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1142901 T3067 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1142908 T3067 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1142916 T3067 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1142918 T3067 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1142919 T3067 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1142920 T3067 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1142922 T3067 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1142923 T3067 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1142924 T3067 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1142924 T3067 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1142925 T3067 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/tempDir-001/control/data/
   [junit4]   2> 1142926 T3067 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16602b3a
   [junit4]   2> 1142927 T3067 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/tempDir-001/control/data
   [junit4]   2> 1142928 T3067 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/tempDir-001/control/data/index/
   [junit4]   2> 1142928 T3067 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1142929 T3067 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/tempDir-001/control/data/index
   [junit4]   2> 1142929 T3067 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=3, maxMergedSegmentMB=66.099609375, floorSegmentMB=1.6201171875, forceMergeDeletesPctAllowed=26.233101335438224, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5436762762407222
   [junit4]   2> 1142930 T3067 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3d16f34a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2dbe9e9b; maxCacheMB=0.587890625 maxMergeSizeMB=0.2783203125)),segFN=segments_1,generation=1}
   [junit4]   2> 1142931 T3067 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1142938 T3067 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1142939 T3067 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1142939 T3067 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1142940 T3067 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1142940 T3067 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1142940 T3067 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1142941 T3067 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1142941 T3067 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1142941 T3067 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1142942 T3067 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1142943 T3067 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1142944 T3067 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1142945 T3067 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1142946 T3067 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1142947 T3067 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1142948 T3067 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1142948 T3067 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1142949 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1142951 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1142952 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1142953 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1142954 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1142955 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1142956 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1142957 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1142958 T3067 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1142958 T3067 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1142992 T3067 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1142996 T3067 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1142999 T3067 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1143003 T3067 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1143006 T3067 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1143012 T3067 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1143013 T3067 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1143014 T3067 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=7.5517578125, floorSegmentMB=0.763671875, forceMergeDeletesPctAllowed=12.40357646257346, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5224481625204392
   [junit4]   2> 1143015 T3067 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@3d16f34a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2dbe9e9b; maxCacheMB=0.587890625 maxMergeSizeMB=0.2783203125)),segFN=segments_1,generation=1}
   [junit4]   2> 1143015 T3067 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1143016 T3067 oass.SolrIndexSearcher.<init> Opening Searcher@7355280f[collection1] main
   [junit4]   2> 1143016 T3067 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1143018 T3067 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1143018 T3067 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1143019 T3067 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1143019 T3067 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1143020 T3067 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1143021 T3067 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1143021 T3067 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1143021 T3067 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1143022 T3067 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1143023 T3068 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7355280f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1143024 T3067 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1143026 T3071 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:56320 collection:control_collection shard:shard1
   [junit4]   2> 1143026 T3037 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 1143027 T3037 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1143027 T3071 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1143032 T3037 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1143033 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1143035 T3071 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1143036 T3075 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@187b32c name:ZooKeeperConnection Watcher:127.0.0.1:56301/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1143036 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1143037 T3037 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1143039 T3063 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1143040 T3071 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1143040 T3037 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1143041 T3064 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1143044 T3063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1143041 T3071 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C7385 name=collection1 org.apache.solr.core.SolrCore@310291ef url=http://127.0.0.1:56320/collection1 node=127.0.0.1:56320_ C7385_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:56320, core=collection1, node_name=127.0.0.1:56320_, state=down}
   [junit4]   2> 1143045 T3071 C7385 P56320 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:56320/collection1/
   [junit4]   2> 1143045 T3075 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 1143046 T3071 C7385 P56320 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1143047 T3071 C7385 P56320 oasc.SyncStrategy.syncToMe http://127.0.0.1:56320/collection1/ has no replicas
   [junit4]   2> 1143047 T3071 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:56320/collection1/ shard1
   [junit4]   2> 1143047 T3071 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1143048 T3037 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1143054 T3076 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1143055 T3064 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56320",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1143162 T3075 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1143207 T3071 oasc.ZkController.register We are http://127.0.0.1:56320/collection1/ and leader is http://127.0.0.1:56320/collection1/
   [junit4]   2> 1143208 T3071 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:56320
   [junit4]   2> 1143208 T3071 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1143209 T3071 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1143209 T3071 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1143211 T3063 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1143214 T3064 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56320",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56320_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1143215 T3064 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56320",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56320_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1143322 T3075 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1143322 T3063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1145453 T3037 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001/cores/collection1
   [junit4]   2> 1145457 T3037 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1145459 T3037 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1145462 T3037 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56334
   [junit4]   2> 1145462 T3037 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@1174ec5
   [junit4]   2> 1145463 T3037 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1145463 T3037 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001
   [junit4]   2> 1145463 T3037 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001/'
   [junit4]   2> 1145516 T3037 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001/solr.xml
   [junit4]   2> 1145569 T3037 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001/cores
   [junit4]   2> 1145570 T3037 oasc.CoreContainer.<init> New CoreContainer 1473179712
   [junit4]   2> 1145570 T3037 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest 9A58E2CFE32BB27D-001/shard-1-001/]
   [junit4]   2> 1145571 T3037 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1145572 T3037 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1145572 T3037 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1145572 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1145573 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1145573 T3037 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1145573 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1145574 T3037 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1145574 T3037 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1145575 T3037 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1145575 T3037 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1145576 T3037 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1145576 T3037 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1145576 T3037 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1145577 T3037 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1145577 T3037 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56301/solr
   [junit4]   2> 1145577 T3037 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1145578 T3037 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1145579 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1145581 T3088 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8b15e38 name:ZooKeeperConnection Watcher:127.0.0.1:56301 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1145582 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1145583 T3037 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1145585 T3037 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1145587 T3091 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23e1f0ab name:ZooKeeperConnection Watcher:127.0.0.1:56301/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1145587 T3037 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1145593 T3037 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1146597 T3037 oasc.ZkController.createEphemeralLiveN

[...truncated too long message...]

se WARN Closing 0 connections to: http://127.0.0.1:36984/cd_ulu/te
   [junit4]   2> 2358988 T9686 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:36996/cd_ulu/te
   [junit4]   2> 2358988 T9686 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:36970/cd_ulu/te
   [junit4]   2> 2358989 T9686 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:36938/cd_ulu/te
   [junit4]   2> 2358989 T9686 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2358989 T9686 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36984/cd_ulu/te
   [junit4]   2> 2358990 T9686 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36996/cd_ulu/te
   [junit4]   2> 2358990 T9686 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36970/cd_ulu/te
   [junit4]   2> 2358990 T9686 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36938/cd_ulu/te
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=9A58E2CFE32BB27D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=is_IS -Dtests.timezone=US/East-Indiana -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   69.6s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36996/cd_ulu/te/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9A58E2CFE32BB27D:120CDD154DD7DF85]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:295)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:209)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36996/cd_ulu/te/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 2359003 T9686 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9A58E2CFE32BB27D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=FSTOrd50, a_t=FSTOrd50, intDefault=FSTOrd50, timestamp=FSTOrd50, _version_=FSTOrd50, multiDefault=FST50, id=PostingsFormat(name=Memory doPackFST= true), text=PostingsFormat(name=Direct)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=is_IS, timezone=US/East-Indiana
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=110332856,total=368574464
   [junit4]   2> NOTE: All tests run in this JVM: [TestFaceting, TestAtomicUpdateErrorCases, AtomicUpdatesTest, CachingDirectoryFactoryTest, MergeStrategyTest, ShardRoutingCustomTest, RequestHandlersTest, HighlighterConfigTest, TestCoreContainer, OverseerTest, FastVectorHighlighterTest, TestManagedSchemaDynamicFieldResource, ZkNodePropsTest, TestSolrIndexConfig, RollingRestartTest, TestMiniSolrCloudCluster, ZkSolrClientTest, TestSchemaResource, MigrateRouteKeyTest, StatelessScriptUpdateProcessorFactoryTest, DistributedFacetPivotWhiteBoxTest, DirectSolrSpellCheckerTest, TestSolrJ, TestCollapseQParserPlugin, TestPartialUpdateDeduplication, TestXIncludeConfig, SolrCmdDistributorTest, CoreAdminCreateDiscoverTest, OpenExchangeRatesOrgProviderTest, TestManagedSchemaFieldResource, TestCloudSchemaless, UUIDFieldTest, OverseerCollectionProcessorTest, TestSolrDeletionPolicy2, PrimUtilsTest, ExternalFileFieldSortTest, RequiredFieldsTest, IndexBasedSpellCheckerTest, DebugComponentTest, RegexBoostProcessorTest, SaslZkACLProviderTest, DOMUtilTest, TestCollationField, TestClassNameShortening, TestComplexPhraseQParserPlugin, DistribCursorPagingTest, TestStressReorder, TestHighlightDedupGrouping, SimpleFacetsTest, AnalysisErrorHandlingTest, SpellCheckCollatorTest, TestUniqueKeyFieldResource, TestReloadAndDeleteDocs, TestUpdate, DistanceFunctionTest, TestFastLRUCache, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestTolerantSearch, AliasIntegrationTest, AssignTest, AsyncMigrateRouteKeyTest, CloudExitableDirectoryReaderTest, CustomCollectionTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, LeaderInitiatedRecoveryOnCommitTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 69.68s, 1 test, 1 error <<< FAILURES!

[...truncated 278 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1847 tests, 5 errors, 1 failure, 55 ignored (8 assumptions)

Total time: 149 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 723576863 bytes
Compression is 0.0%
Took 2 min 14 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 747 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/747/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Error from server at https://127.0.0.1:32728: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:905)  at org.apache.solr.cloud.ZkController.register(ZkController.java:809)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.server.ssl.SslSocketConnector$SslConnectorEndPoint.run(SslSocketConnector.java:670)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /admin/cores. Reason: <pre>    {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:905)  at org.apache.solr.cloud.ZkController.register(ZkController.java:809)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.server.ssl.SslSocketConnector$SslConnectorEndPoint.run(SslSocketConnector.java:670)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:32728: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:905)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:809)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.server.ssl.SslSocketConnector$SslConnectorEndPoint.run(SslSocketConnector.java:670)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /admin/cores. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:905)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:809)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.server.ssl.SslSocketConnector$SslConnectorEndPoint.run(SslSocketConnector.java:670)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:527)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:329)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:446)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:193)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:462)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:462)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51979/lk/xo/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51979/lk/xo/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:295)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:209)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10047 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/init-core-data-001
   [junit4]   2> 23540 T24 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 23541 T24 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 23596 T24 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 23609 T25 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 24112 T24 oasc.ZkTestServer.run start zk server on port:51693
   [junit4]   2> 24279 T24 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 24576 T24 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 24798 T32 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a2a9206 name:ZooKeeperConnection Watcher:127.0.0.1:51693 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 24799 T24 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 24808 T24 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 24825 T24 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 24976 T24 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 24977 T24 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 24979 T35 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60405aae name:ZooKeeperConnection Watcher:127.0.0.1:51693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 24980 T24 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 24980 T24 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 25013 T24 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 25017 T24 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 25021 T24 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 25024 T24 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 25028 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 25068 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 25074 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 25075 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 25079 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 25080 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 25085 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 25086 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 25090 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 25091 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 25095 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 25096 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 25100 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 25102 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 25107 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 25109 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 25115 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 25116 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 25122 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 25123 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 25129 T24 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 25130 T24 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 25169 T24 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 25170 T24 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 25173 T38 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ee5c773 name:ZooKeeperConnection Watcher:127.0.0.1:51693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 25174 T24 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 25175 T24 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 27210 T24 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1
   [junit4]   2> 28400 T24 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 29068 T24 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 29127 T24 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:62516
   [junit4]   2> 29479 T24 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 29480 T24 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 29480 T24 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001
   [junit4]   2> 29481 T24 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/'
   [junit4]   2> 29549 T24 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/solr.xml
   [junit4]   2> 29669 T24 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores
   [junit4]   2> 29670 T24 oasc.CoreContainer.<init> New CoreContainer 232426262
   [junit4]   2> 29670 T24 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/]
   [junit4]   2> 29787 T24 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 29788 T24 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 29788 T24 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 29789 T24 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 29789 T24 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 29790 T24 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 29790 T24 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 29791 T24 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 29791 T24 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 29792 T24 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 29792 T24 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 29797 T24 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 29800 T24 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 29801 T24 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 29801 T24 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 29802 T24 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51693/solr
   [junit4]   2> 29867 T24 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 29868 T24 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 29869 T24 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29873 T51 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b0cfe06 name:ZooKeeperConnection Watcher:127.0.0.1:51693 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29873 T24 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29874 T24 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 29904 T24 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29906 T54 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@264ccd21 name:ZooKeeperConnection Watcher:127.0.0.1:51693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29906 T24 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29931 T24 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 29955 T24 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 29965 T24 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 29969 T24 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 29974 T24 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 29999 T24 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 30001 T24 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62516_
   [junit4]   2> 30009 T24 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62516_
   [junit4]   2> 30014 T24 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 30017 T24 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 30020 T24 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 30054 T24 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62516_
   [junit4]   2> 30055 T24 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 30058 T24 oasc.Overseer.start Overseer (id=93222363263270916-127.0.0.1:62516_-n_0000000000) starting
   [junit4]   2> 30089 T24 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 30138 T24 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 30177 T56 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 30178 T24 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 30182 T24 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 30185 T24 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 30224 T55 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 30225 T24 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores
   [junit4]   2> 30264 T24 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/
   [junit4]   2> 30265 T24 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 30268 T58 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 30269 T58 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 30288 T54 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 30288 T58 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 30299 T55 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:62516_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:62516",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 30365 T55 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:62516_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:62516",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 30373 T55 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 30425 T55 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 30448 T54 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 31290 T58 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 31290 T58 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 31292 T58 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 31293 T58 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 31295 T58 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 31296 T58 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/'
   [junit4]   2> 31354 T58 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 31383 T58 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 31437 T58 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 31529 T58 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 31656 T58 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31659 T58 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 31778 T58 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 32403 T58 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 32428 T58 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 32432 T58 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 32479 T58 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32489 T58 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32497 T58 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32500 T58 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32500 T58 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32501 T58 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32503 T58 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32504 T58 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32504 T58 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 32505 T58 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 32516 T58 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/data/
   [junit4]   2> 32535 T58 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@737ad430
   [junit4]   2> 32736 T58 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/data
   [junit4]   2> 32737 T58 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/data/index/
   [junit4]   2> 32738 T58 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 32738 T58 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 23241A13BACB9CEF-001/control-001/cores/collection1/data/index
   [junit4]   2> 32739 T58 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4292099206988942]
   [junit4]   2> 32764 T58 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fc55ca0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16443d7f),segFN=segments_1,generation=1}
   [junit4]   2> 32765 T58 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 32784 T58 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 32982 T58 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 33004 T58 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 33005 T58 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33006 T58 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33006 T58 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 33287 T58 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33288 T58 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33288 T58 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 33302 T58 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 33303 T58 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 33305 T58 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 33306 T58 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 33308 T58 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 33310 T58 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 33312 T58 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 33313 T58 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 33315 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 33317 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 33319 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 33320 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 33322 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 33324 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 33325 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 33327 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 33329 T58 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 33388 T58 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 33422 T58 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33425 T58 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33428 T58 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33431 T58 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33441 T58 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 33575 T58 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 33576 T58 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 33578 T58 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=4, maxMergedSegmentMB=9.7890625, floorSegmentMB=1.5751953125, forceMergeDeletesPctAllowed=5.408223721707487, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13806594386289
   [junit4]   2> 33588 T58 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fc55ca0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16443d7f),segFN=segments_1,generation=1}
   [junit4]   2> 33589 T58 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 33589 T58 oass.SolrIndexSearcher.<init> Opening Searcher@69da98bf[collection1] main
   [junit4]   2> 33590 T58 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 33592 T58 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 33593 T58 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 33610 T58 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 33611 T58 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 33611 T58 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 33612 T58 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 33613 T58 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 33614 T58 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33657 T58 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33661 T59 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@69da98bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 33671 T58 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 33672 T62 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:62516 collection:control_collection shard:shard1
   [junit4]   2> 33673 T24 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 33674 T24 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 33729 T62 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 33741 T62 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 33744 T54 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 33745 T62 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 33746 T62 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@7227abfc url=https://127.0.0.1:62516/collection1 node=127.0.0.1:62516_ C1_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:62516_, state=down, core=collection1, base_url=https://127.0.0.1:62516}
   [junit4]   2> 33746 T62 C1 P62516 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:62516/collection1/
   [junit4]   2> 33747 T55 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 

[...truncated too long message...]

   2> 1892240 T4012 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63813/
   [junit4]   2> 1892241 T4012 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63773/
   [junit4]   2> 1892241 T4012 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63792/
   [junit4]   2> 1892241 T4012 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63804/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=23241A13BACB9CEF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_TW -Dtests.timezone=Asia/Khandyga -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    119s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([23241A13BACB9CEF:AB7025C91437F117]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:295)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:209)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1892251 T4012 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 23241A13BACB9CEF-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=Memory doPackFST= true), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=PostingsFormat(name=Memory doPackFST= true), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Memory doPackFST= true), text=FST50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=zh_TW, timezone=Asia/Khandyga
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=238792736,total=493355008
   [junit4]   2> NOTE: All tests run in this JVM: [IgnoreCommitOptimizeUpdateProcessorFactoryTest, JSONWriterTest, TestShardHandlerFactory, TestLMJelinekMercerSimilarityFactory, ConvertedLegacyTest, DistanceUnitsTest, DocumentAnalysisRequestHandlerTest, DistributedQueryComponentCustomSortTest, TestSolrXml, TestFunctionQuery, EchoParamsTest, TestBlobHandler, BJQParserTest, TestSchemaNameResource, DistributedFacetPivotLargeTest, TestLeaderElectionZkExpiry, TestSweetSpotSimilarityFactory, ResourceLoaderTest, TestFastOutputStream, AsyncMigrateRouteKeyTest, TermVectorComponentTest, DisMaxRequestHandlerTest, TestRecovery, ZkControllerTest, TestRecoveryHdfs, TestConfigReload, CloudExitableDirectoryReaderTest, BasicZkTest, ConnectionManagerTest, IndexSchemaTest, TestStressRecovery, TestInfoStreamLogging, SystemInfoHandlerTest, UniqFieldsUpdateProcessorFactoryTest, TestExceedMaxTermLength, ChangedSchemaMergeTest, SortByFunctionTest, CollectionsAPIAsyncDistributedZkTest, SuggesterTSTTest, OpenCloseCoreStressTest, TestObjectReleaseTracker, TestZkChroot, WordBreakSolrSpellCheckerTest, SuggesterFSTTest, SolrInfoMBeanTest, SharedFSAutoReplicaFailoverUtilsTest, TestBinaryResponseWriter, HttpPartitionTest, PingRequestHandlerTest, TestDocumentBuilder, AnalyticsQueryTest, HighlighterTest, CircularListTest, AutoCommitTest, TestSolrDeletionPolicy1, TestStressVersions, TestDistributedGrouping, TestRangeQuery, TestGroupingSearch, DirectUpdateHandlerTest, TestIndexSearcher, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SolrRequestParserTest, TestStressLucene, SuggesterWFSTTest, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, DocValuesMultiTest, DebugComponentTest, TestQueryUtils, TestWriterPerf, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, TestSolrQueryParser, TestJmxIntegration, JsonLoaderTest, TestComponentsName, CopyFieldTest, BadComponentTest, NumericFieldsTest, TestFuzzyAnalyzedSuggestions, TestLuceneMatchVersion, TestDFRSimilarityFactory, TestFastWriter, OpenExchangeRatesOrgProviderTest, PreAnalyzedFieldTest, TestUtils, UUIDFieldTest, CursorPagingTest, DeleteInactiveReplicaTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 118.97s, 1 test, 1 error <<< FAILURES!

[...truncated 337 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1847 tests, 6 errors, 55 ignored (8 assumptions)

Total time: 180 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 672824374 bytes
Compression is 0.0%
Took 2 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 746 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/746/

6 tests failed.
REGRESSION:  org.apache.solr.search.TestSearcherReuse.test

Error Message:
expected same:<Searcher@5096d4e4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c1) Uninverting(_1(5.1.0):c2) Uninverting(_2(5.1.0):c1) Uninverting(_3(5.1.0):c1)))}> was not:<Searcher@4d8a4268[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(5.1.0):c5)))}>

Stack Trace:
java.lang.AssertionError: expected same:<Searcher@5096d4e4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c1) Uninverting(_1(5.1.0):c2) Uninverting(_2(5.1.0):c1) Uninverting(_3(5.1.0):c1)))}> was not:<Searcher@4d8a4268[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(5.1.0):c5)))}>
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotSame(Assert.java:641)
	at org.junit.Assert.assertSame(Assert.java:580)
	at org.junit.Assert.assertSame(Assert.java:593)
	at org.apache.solr.search.TestSearcherReuse.assertSearcherHasNotChanged(TestSearcherReuse.java:247)
	at org.apache.solr.search.TestSearcherReuse.test(TestSearcherReuse.java:104)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:44413/tm/r/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44413/tm/r/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:175)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:153)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:133)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:123)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:109)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:299)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:213)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9859 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/init-core-data-001
   [junit4]   2> 465930 T2183 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /tm/r
   [junit4]   2> 465936 T2183 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 465937 T2184 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 466037 T2183 oasc.ZkTestServer.run start zk server on port:44319
   [junit4]   2> 466038 T2183 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 466039 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 466044 T2191 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b30dee name:ZooKeeperConnection Watcher:127.0.0.1:44319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 466044 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 466044 T2183 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 466045 T2183 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 466047 T2183 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 466048 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 466049 T2194 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7307ac43 name:ZooKeeperConnection Watcher:127.0.0.1:44319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 466050 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 466050 T2183 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 466050 T2183 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 466053 T2183 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 466054 T2183 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 466056 T2183 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 466058 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 466058 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 466061 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 466062 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 466165 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 466166 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 466167 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 466168 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 466170 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 466171 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 466172 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 466173 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 466175 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 466175 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 466177 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 466178 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 466179 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 466180 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 466182 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 466182 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 466184 T2183 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 466184 T2183 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 467711 T2183 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores/collection1
   [junit4]   2> 467716 T2183 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 467719 T2183 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:44328
   [junit4]   2> 467720 T2183 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 467720 T2183 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 467720 T2183 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001
   [junit4]   2> 467721 T2183 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/'
   [junit4]   2> 467770 T2183 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/solr.xml
   [junit4]   2> 467833 T2183 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores
   [junit4]   2> 467834 T2183 oasc.CoreContainer.<init> New CoreContainer 915144564
   [junit4]   2> 467834 T2183 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/]
   [junit4]   2> 467835 T2183 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 467836 T2183 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 467836 T2183 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 467836 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 467836 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 467837 T2183 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 467837 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 467837 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 467838 T2183 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 467838 T2183 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 467838 T2183 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 467839 T2183 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 467839 T2183 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 467839 T2183 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 467840 T2183 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 467840 T2183 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44319/solr
   [junit4]   2> 467840 T2183 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 467840 T2183 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 467841 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 467843 T2206 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fafc5c1 name:ZooKeeperConnection Watcher:127.0.0.1:44319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 467844 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 467845 T2183 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 467847 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 467848 T2209 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af85746 name:ZooKeeperConnection Watcher:127.0.0.1:44319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 467848 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 467850 T2183 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 467852 T2183 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 467854 T2183 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 467856 T2183 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 467857 T2183 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 467859 T2183 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 467861 T2183 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44327_tm%2Fr
   [junit4]   2> 467861 T2183 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44327_tm%2Fr
   [junit4]   2> 467863 T2183 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 467864 T2183 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 467866 T2183 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 467867 T2183 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:44327_tm%2Fr
   [junit4]   2> 467868 T2183 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 467869 T2183 oasc.Overseer.start Overseer (id=93216621856096259-127.0.0.1:44327_tm%2Fr-n_0000000000) starting
   [junit4]   2> 467871 T2183 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 467877 T2183 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 467878 T2211 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 467879 T2183 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 467880 T2183 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 467882 T2183 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 467884 T2210 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 467886 T2183 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores
   [junit4]   2> 467889 T2183 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores/collection1/
   [junit4]   2> 467889 T2183 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 467890 T2213 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 467890 T2213 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 467891 T2213 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 467891 T2209 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 467894 T2210 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44327_tm%2Fr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44327/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 467894 T2210 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44327_tm%2Fr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44327/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 467894 T2210 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 467895 T2210 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 467896 T2209 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 468891 T2213 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 468892 T2213 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 468892 T2213 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 468893 T2213 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 468893 T2213 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 468893 T2213 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores/collection1/'
   [junit4]   2> 468910 T2213 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 468912 T2213 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 468923 T2213 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 468942 T2213 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 468974 T2213 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 469076 T2213 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 469083 T2213 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 469531 T2213 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 469546 T2213 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 469550 T2213 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 469561 T2213 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 469566 T2213 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 469571 T2213 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 469572 T2213 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 469573 T2213 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 469573 T2213 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 469574 T2213 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 469574 T2213 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 469574 T2213 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 469574 T2213 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 469575 T2213 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/control/data/
   [junit4]   2> 469575 T2213 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@cc89dad
   [junit4]   2> 469576 T2213 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/control/data
   [junit4]   2> 469576 T2213 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/control/data/index/
   [junit4]   2> 469576 T2213 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 469577 T2213 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/control/data/index
   [junit4]   2> 469577 T2213 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=54.0, floorSegmentMB=1.642578125, forceMergeDeletesPctAllowed=29.43128436176714, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 469578 T2213 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@67d68f53 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@375bc767),segFN=segments_1,generation=1}
   [junit4]   2> 469579 T2213 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 469585 T2213 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 469586 T2213 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 469586 T2213 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 469586 T2213 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 469587 T2213 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 469587 T2213 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 469587 T2213 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 469588 T2213 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 469588 T2213 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 469589 T2213 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 469589 T2213 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 469590 T2213 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 469590 T2213 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 469591 T2213 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 469591 T2213 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 469592 T2213 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 469592 T2213 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 469593 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 469594 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 469594 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 469595 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 469595 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 469596 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 469596 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 469597 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 469597 T2213 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 469597 T2213 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 469618 T2213 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 469620 T2213 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 469622 T2213 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 469624 T2213 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 469626 T2213 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 469630 T2213 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 469630 T2213 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 469631 T2213 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 469631 T2213 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@67d68f53 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@375bc767),segFN=segments_1,generation=1}
   [junit4]   2> 469632 T2213 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 469632 T2213 oass.SolrIndexSearcher.<init> Opening Searcher@58ec77ee[collection1] main
   [junit4]   2> 469632 T2213 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 469633 T2213 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 469633 T2213 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 469634 T2213 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 469634 T2213 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 469634 T2213 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 469635 T2213 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 469635 T2213 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 469635 T2213 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 469635 T2213 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 469636 T2214 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@58ec77ee[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 469637 T2213 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 469638 T2217 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:44327/tm/r collection:control_collection shard:shard1
   [junit4]   2> 469638 T2183 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 469639 T2183 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 469639 T2217 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 469642 T2183 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 469642 T2217 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 469643 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 469644 T2209 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 469644 T2217 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 469644 T2221 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17ef5986 name:ZooKeeperConnection Watcher:127.0.0.1:44319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 469644 T2210 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 469644 T2217 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 469644 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C337 name=collection1 org.apache.solr.core.SolrCore@621649e3 url=http://127.0.0.1:44327/tm/r/collection1 node=127.0.0.1:44327_tm%2Fr C337_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:44327_tm%2Fr, base_url=http://127.0.0.1:44327/tm/r, core=collection1, state=down}
   [junit4]   2> 469645 T2217 C337 P44327 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:44327/tm/r/collection1/
   [junit4]   2> 469645 T2183 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 469646 T2209 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 469645 T2217 C337 P44327 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 469646 T2217 C337 P44327 oasc.SyncStrategy.syncToMe http://127.0.0.1:44327/tm/r/collection1/ has no replicas
   [junit4]   2> 469646 T2183 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 469646 T2217 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:44327/tm/r/collection1/ shard1
   [junit4]   2> 469647 T2217 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 469648 T2183 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 469650 T2222 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 469650 T2210 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44327/tm/r",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 469753 T2221 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 469802 T2217 oasc.ZkController.register We are http://127.0.0.1:44327/tm/r/collection1/ and leader is http://127.0.0.1:44327/tm/r/collection1/
   [junit4]   2> 469803 T2217 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44327/tm/r
   [junit4]   2> 469803 T2217 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 469803 T2217 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 469803 T2217 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 469805 T2209 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 469806 T2210 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:44327_tm%2Fr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44327/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 469807 T2210 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:44327_tm%2Fr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44327/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 469910 T2221 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 469910 T2209 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 470644 T2183 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores/collection1
   [junit4]   2> 470646 T2183 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 470647 T2183 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 470649 T2183 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:44363
   [junit4]   2> 470649 T2183 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 470650 T2183 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 470650 T2183 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001
   [junit4]   2> 470650 T2183 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/'
   [junit4]   2> 470679 T2183 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/solr.xml
   [junit4]   2> 470726 T2183 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores
   [junit4]   2> 470726 T2183 oasc.CoreContainer.<init> New CoreContainer 616315957
   [junit4]   2> 470727 T2183 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/]
   [junit4]   2> 470728 T2183 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 470728 T2183 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 470728 T2183 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 470728 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 470729 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 470729 T2183 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 470729 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 470729 T2183 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 470729 T2183 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 470730 T2183 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 470730 T2183 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 470730 T2183 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 470731 T2183 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 470731 T2183 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 470731 T2183 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 470731 T2183 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44319/solr
   [junit4]   2> 470732 T2183 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 470732 T2183 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 470732 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 470734 T2234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7327c2a9 name:ZooKeeperConnection Watcher:127.0.0.1:44319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 470735 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 470735 T2183 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 470737 T2183 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 470738 T2237 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a18196e name:ZooKeeperConnection Watcher:127.0.0.1:44319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 470738 T2183 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 470741 T2183 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 471745 T2183 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44362_tm%2Fr
   [junit4]   2> 471746 T2183 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44362_tm%2Fr
   [junit4]   2> 471749 T2183 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 471751 T2183 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores
   [junit4]   2> 471752 T2183 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores/collection1/
   [junit4]   2> 471753 T2183 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 471753 T2238 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 471754 T2238 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 471755 T2238 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 471755 T2209 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 471756 T2210 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44362_tm%2Fr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44362/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 471756 T2210 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44362_tm%2Fr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44362/tm/r",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 471756 T2210 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 471757 T2210 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 471758 T2237 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 471758 T2221 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 471758 T2209 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 472755 T2238 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 472756 T2238 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 472756 T2238 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 472756 T2238 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 472757 T2238 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 472757 T2238 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores/collection1/'
   [junit4]   2> 472771 T2238 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 472773 T2238 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 472783 T2238 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 472801 T2238 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 472839 T2238 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 472941 T2238 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 472949 T2238 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 473247 T2238 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 473255 T2238 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 473257 T2238 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 473265 T2238 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 473269 T2238 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 473272 T2238 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 473273 T2238 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 473274 T2238 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 473274 T2238 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 473274 T2238 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 473275 T2238 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 473275 T2238 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 473275 T2238 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 473275 T2238 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9AF8E4923DC47371-001/tempDir-001/jetty1/
   [junit4]   2> 473275 T2238 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@cc89dad
   [junit4]   2> 473276 T2238 oa

[...truncated too long message...]

42[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2289435 T10531 C2928 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 7
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherReuse -Dtests.method=test -Dtests.seed=9AF8E4923DC47371 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=lv_LV -Dtests.timezone=Etc/GMT+0 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.18s J3 | TestSearcherReuse.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected same:<Searcher@5096d4e4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):c1) Uninverting(_1(5.1.0):c2) Uninverting(_2(5.1.0):c1) Uninverting(_3(5.1.0):c1)))}> was not:<Searcher@4d8a4268[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_4(5.1.0):c5)))}>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9AF8E4923DC47371:12ACDB4893381E89]:0)
   [junit4]    > 	at org.apache.solr.search.TestSearcherReuse.assertSearcherHasNotChanged(TestSearcherReuse.java:247)
   [junit4]    > 	at org.apache.solr.search.TestSearcherReuse.test(TestSearcherReuse.java:104)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2289446 T10531 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 2289446 T10531 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1201313495
   [junit4]   2> 2289447 T10531 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@29640bf0
   [junit4]   2> 2289447 T10531 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=6,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=486,transaction_logs_total_number=5}
   [junit4]   2> 2289448 T10531 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2289448 T10531 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2289448 T10531 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2289449 T10531 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2289450 T10531 oasc.CachingDirectoryFactory.close Closing NRTCachingDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2289450 T10531 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 2289450 T10531 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001/index
   [junit4]   2> 2289450 T10531 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001;done=false>>]
   [junit4]   2> 2289451 T10531 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.search.TestSearcherReuse 9AF8E4923DC47371-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=lv_LV, timezone=Etc/GMT+0
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=250862792,total=529006592
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteLastCustomShardedReplicaTest, ClusterStateUpdateTest, AddSchemaFieldsUpdateProcessorFactoryTest, UpdateRequestProcessorFactoryTest, TestPostingsSolrHighlighter, TestHighFrequencyDictionaryFactory, HdfsLockFactoryTest, OverriddenZkACLAndCredentialsProvidersTest, LeaderElectionTest, TestImplicitCoreProperties, TestTolerantSearch, TestReRankQParserPlugin, ShardRoutingTest, AlternateDirectoryTest, TestOverriddenPrefixQueryForCustomFieldType, TestManagedSchema, ExternalCollectionsTest, TestManagedResourceStorage, DateMathParserTest, TestFieldResource, TestExactSharedStatsCache, TestCloudPivotFacet, TestValueSourceCache, ZkCLITest, TestOrdValues, TestFiltering, BadIndexSchemaTest, AnalysisAfterCoreReloadTest, HighlighterMaxOffsetTest, DocValuesMultiTest, TestCopyFieldCollectionResource, TestStandardQParsers, TestWriterPerf, ClusterStateTest, CurrencyFieldXmlFileTest, OutputWriterTest, UnloadDistributedZkTest, SampleTest, TestBlendedInfixSuggestions, TestCSVLoader, LeaderInitiatedRecoveryOnCommitTest, BufferStoreTest, IndexSchemaRuntimeFieldTest, TestDynamicFieldCollectionResource, TestLMDirichletSimilarityFactory, TestUtils, DefaultValueUpdateProcessorTest, DocValuesTest, TestSchemaVersionResource, TestJmxMonitoredMap, SolrCoreTest, TestIndexingPerformance, ReplicationFactorTest, TestSurroundQueryParser, ResponseLogComponentTest, CollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, TestDistributedGrouping, TestHashPartitioner, TestRealTimeGet, TestStressVersions, TestRangeQuery, SpellCheckComponentTest, TestGroupingSearch, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, TestExtendedDismaxParser, CoreAdminHandlerTest, SuggesterWFSTTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestAtomicUpdateErrorCases, DirectUpdateHandlerOptimizeTest, SortByFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, CacheHeaderTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TermsComponentTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, JSONWriterTest, JsonLoaderTest, CSVRequestHandlerTest, TestBinaryResponseWriter, SOLR749Test, TestSolrIndexConfig, CopyFieldTest, TestStressRecovery, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, TestSolrJ, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, FileUtilsTest, CircularListTest, TestRTGBase, TestDistributedMissingSort, TestHighlightDedupGrouping, AliasIntegrationTest, AsyncMigrateRouteKeyTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, HttpPartitionTest, OverseerStatusTest, RemoteQueryErrorTest, SSLMigrationTest, ShardSplitTest, DistributedExpandComponentTest, DistributedFacetPivotSmallAdvancedTest, DistributedFacetPivotSmallTest, DistributedFacetPivotWhiteBoxTest, DistributedQueryComponentCustomSortTest, DistributedSuggestComponentTest, TestDefaultSearchFieldResource, TestSerializedLuceneMatchVersion, TestSolrQueryParserDefaultOperatorResource, ChangedSchemaMergeTest, DocValuesMissingTest, EnumFieldTest, TestBulkSchemaConcurrent, TestCollationFieldDocValues, CursorMarkTest, MergeStrategyTest, RankQueryTest, TestComplexPhraseQParserPlugin, TestCustomSort, TestInitQParser, TestMaxScoreQueryParser, TestMissingGroups, TestSearcherReuse]
   [junit4] Completed on J3 in 0.68s, 1 test, 1 failure <<< FAILURES!

[...truncated 105 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1847 tests, 5 errors, 1 failure, 55 ignored (8 assumptions)

Total time: 154 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 673444131 bytes
Compression is 0.0%
Took 2 min 7 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 745 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/745/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:47781: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:906)  at org.apache.solr.cloud.ZkController.register(ZkController.java:810)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /admin/cores. Reason: <pre>    {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:906)  at org.apache.solr.cloud.ZkController.register(ZkController.java:810)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:47781: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:906)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:810)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /admin/cores. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:906)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:810)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:527)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:338)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:457)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:193)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Error executing query

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Error executing query
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:98)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:308)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:411)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.ParseException: Invalid content type: 
	at org.apache.http.entity.ContentType.parse(ContentType.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:511)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	... 47 more


REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:1F687DFBF763524A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49211/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49211/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49211/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50726/ueph/db/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50726/ueph/db/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50726/ueph/db/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32343/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32343/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:175)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:153)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:133)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48382/o_/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48382/o_/collection1
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:123)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:109)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48382/o_/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28994/vp/f/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28994/vp/f/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:299)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:213)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28994/vp/f/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9522 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> log4j:WARN No such property [conversionPattern] in org.apache.solr.util.SolrLogLayout.
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001
   [junit4]   2> 4311 T11 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 4552 T11 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 5188 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 5199 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 5200 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 5202 T11 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 6622 T11 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 6863 T11 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 7415 T11 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 7472 T11 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 7620 T11 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 11495 T11 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 11586 T11 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 11601 T11 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 13258 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 13300 T11 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 13308 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 13318 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 13318 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 13319 T11 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 13321 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 13321 T11 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 13401 T11 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 13402 T11 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 13403 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 13532 T11 oasc.CoreContainer.<init> New CoreContainer 276237606
   [junit4]   2> 13533 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 13602 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 13603 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 13614 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 13615 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 13616 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 13616 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 13617 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 13618 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 13618 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 13619 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 13619 T11 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 14478 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 14519 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 14529 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 14543 T11 oasc.CoreContainer.load Host Name: null
   [junit4]   2> 14919 T12 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 14922 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 14923 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 14925 T12 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 15131 T12 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 15189 T12 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 15280 T12 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 15282 T12 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 15366 T12 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 16004 T12 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 16035 T12 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 16040 T12 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 16085 T12 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 16095 T12 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 16103 T12 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 16105 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 16106 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 16107 T12 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 16109 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 16109 T12 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 16129 T12 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 16615 T12 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 16659 T12 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001/
   [junit4]   2> 16668 T12 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 16837 T12 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001
   [junit4]   2> 16838 T12 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001/index/
   [junit4]   2> 16839 T12 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 16871 T12 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 4EBC847B461062ED-001/init-core-data-001/index
   [junit4]   2> 16875 T12 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23522477216749255]
   [junit4]   2> 17233 T12 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49480586 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1762c600),segFN=segments_1,generation=1}
   [junit4]   2> 17240 T12 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 17334 T12 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 19802 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 20042 T12 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 20043 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 20044 T12 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 20045 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 20046 T12 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 20046 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 20093 T12 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 20094 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 20346 T12 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 20347 T12 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 20348 T12 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 20506 T12 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20507 T12 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20508 T12 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20509 T12 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 20511 T12 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 20512 T12 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 20578 T12 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 20593 T12 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 20595 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 20597 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 20599 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 20600 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 20601 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 20602 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 20603 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 20604 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 20682 T12 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 20929 T12 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 20932 T12 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 21029 T12 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 21030 T12 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 21031 T12 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 21042 T12 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 21043 T12 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21043 T12 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21044 T12 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21045 T12 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21046 T12 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21047 T12 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21048 T12 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21048 T12 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 21126 T12 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 21203 T12 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 21204 T12 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 21205 T12 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 21635 T12 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21699 T12 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21703 T12 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21707 T12 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 21721 T12 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 21725 T12 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 22003 T12 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 22004 T12 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 22005 T12 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=84.3359375, floorSegmentMB=1.6533203125, forceMergeDeletesPctAllowed=9.871743212457682, segmentsPerTier=25.0, maxCFSSegmentSizeMB=0.361328125, noCFSRatio=0.0
   [junit4]   2> 22033 T12 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49480586 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1762c600),segFN=segments_1,generation=1}
   [junit4]   2> 22033 T12 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 22375 T12 oass.SolrIndexSearcher.<init> Opening Searcher@3ced8fe2[collection1] main
   [junit4]   2> 22520 T12 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 22756 T12 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 22757 T12 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 22758 T12 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 23394 T12 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 24231 T12 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24232 T13 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 24236 T13 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 24236 T13 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 24237 T13 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 24240 T13 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 24242 T12 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24243 T13 oashc.SpellCheckComponent$SpellCheckerListener.ne

[...truncated too long message...]

134 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37743/
   [junit4]   2> 1979820 T9134 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32343/
   [junit4]   2> 1979820 T9134 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20105/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=4EBC847B461062ED -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ko -Dtests.timezone=MST -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   79.6s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32343/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4EBC847B461062ED:C6E8BBA1E8EC0F15]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:175)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:133)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 49 more
   [junit4]   2> 1979833 T9134 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4EBC847B461062ED-001
   [junit4]   2> 79625 T9133 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ko, timezone=MST
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=312334976,total=490733568
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsDirectoryTest, TestLRUStatsCache, TestAnalyzeInfixSuggestions, OutOfBoxZkACLAndCredentialsProvidersTest, MoreLikeThisHandlerTest, URLClassifyProcessorTest, SimpleCollectionCreateDeleteTest, DeleteShardTest, PreAnalyzedUpdateProcessorTest, TestRebalanceLeaders, TestFieldTypeCollectionResource, DocumentBuilderTest, TestReload, SliceStateTest, TestLuceneMatchVersion, TestCollationFieldDocValues, SolrIndexSplitterTest, DateRangeFieldTest, TestRandomDVFaceting, TestReplicationHandlerBackup, TestPerFieldSimilarity, LukeRequestHandlerTest, SpellCheckComponentTest, TestAnalyzedSuggestions, TestExtendedDismaxParser, TestRemoteStreaming, SyncSliceTest, TestRTGBase, SimpleMLTQParserTest, CacheHeaderTest, TestSolrQueryParser, NoCacheHeaderTest, TriLevelCompositeIdRoutingTest, DeleteReplicaTest, TestComponentsName, TestSchemaManager, TestHashPartitioner, UUIDUpdateProcessorFallbackTest, SynonymTokenizerTest, TestSolrDynamicMBean, TestBinaryField, TestNonDefinedSimilarityFactory, TestBulkSchemaAPI, TestFileDictionaryLookup, TestBM25SimilarityFactory, CloudMLTQParserTest, DirectUpdateHandlerOptimizeTest, DistributedQueueTest, TestQuerySenderNoQuery, TestReversedWildcardFilterFactory, SOLR749Test, PeerSyncTest, TestFoldingMultitermQuery, TestRealTimeGet, StressHdfsTest, QueryEqualityTest, FileBasedSpellCheckerTest, ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, AutoCommitTest, TermVectorComponentDistributedTest, TestStressReorder, HardAutoCommitTest, DistributedTermsComponentTest, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, StatsComponentTest, BadIndexSchemaTest, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, OverseerCollectionProcessorTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, SuggesterFSTTest, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, TestTrie, SpatialFilterTest, TestCSVLoader, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, DebugComponentTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, StandardRequestHandlerTest, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TestOmitPositions, RequestHandlersTest, TermVectorComponentTest, TestIndexingPerformance, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestArbitraryIndexDir, FastVectorHighlighterTest, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, ReturnFieldsTest, JSONWriterTest, TestCSVResponseWriter, MBeansHandlerTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestLFUCache, UpdateParamsTest, ResponseLogComponentTest, TestMergePolicyConfig, SampleTest, NumericFieldsTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestCharFilters, TestSweetSpotSimilarityFactory, TestFastWriter, TimeZoneUtilsTest, ChaosMonkeyNothingIsSafeTest, ExternalCollectionsTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 79.92s, 1 test, 1 error <<< FAILURES!

[...truncated 367 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1846 tests, 7 errors, 1 failure, 55 ignored (8 assumptions)

Total time: 162 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 508331656 bytes
Compression is 0.0%
Took 1 min 35 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 744 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/744/

7 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
shard3 is not consistent.  Got 74 from https://127.0.0.1:43403/_cv/i/collection1lastClient and got 79 from https://127.0.0.1:43422/_cv/i/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 74 from https://127.0.0.1:43403/_cv/i/collection1lastClient and got 79 from https://127.0.0.1:43422/_cv/i/collection1
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1246)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1225)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:159)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1422237634958,generation=2,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qlv.cfe, _qlv.cfs, _qlv.si, _qlw.cfe, _qlw.cfs, _qlw.si, _qlx.cfe, _qlx.cfs, _qlx.si, _qly.cfe, _qly.cfs, _qly.si, _qlz.cfe, _qlz.cfs, _qlz.si, _qm0.cfe, _qm0.cfs, _qm0.si, _qm1.cfe, _qm1.cfs, _qm1.si, _qm2.cfe, _qm2.cfs, _qm2.si, _qm3.cfe, _qm3.cfs, _qm3.si, _qm4.cfe, _qm4.cfs, _qm4.si, _qm5.cfe, _qm5.cfs, _qm5.si, _qm6.cfe, _qm6.cfs, _qm6.si, _qm7.cfe, _qm7.cfs, _qm7.si, _qm8.cfe, _qm8.cfs, _qm8.si, _qm9.cfe, _qm9.cfs, _qm9.si, _qma.cfe, _qma.cfs, _qma.si, _qmb.cfe, _qmb.cfs, _qmb.si, _qmc.cfe, _qmc.cfs, _qmc.si, _qmd.cfe, _qmd.cfs, _qmd.si, _qme.cfe, _qme.cfs, _qme.si, _qmf.cfe, _qmf.cfs, _qmf.si, _qmg.cfe, _qmg.cfs, _qmg.si, _qmh.cfe, _qmh.cfs, _qmh.si, _qmi.cfe, _qmi.cfs, _qmi.si, _qmj.cfe, _qmj.cfs, _qmj.si, _qmk.cfe, _qmk.cfs, _qmk.si, _qml.cfe, _qml.cfs, _qml.si, _qmm.cfe, _qmm.cfs, _qmm.si, _qmn.cfe, _qmn.cfs, _qmn.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_2]}]> but was:<[{indexVersion=1422237634958,generation=3,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmp.cfe, _qmp.cfs, _qmp.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_3]}, {indexVersion=1422237634958,generation=2,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qlv.cfe, _qlv.cfs, _qlv.si, _qlw.cfe, _qlw.cfs, _qlw.si, _qlx.cfe, _qlx.cfs, _qlx.si, _qly.cfe, _qly.cfs, _qly.si, _qlz.cfe, _qlz.cfs, _qlz.si, _qm0.cfe, _qm0.cfs, _qm0.si, _qm1.cfe, _qm1.cfs, _qm1.si, _qm2.cfe, _qm2.cfs, _qm2.si, _qm3.cfe, _qm3.cfs, _qm3.si, _qm4.cfe, _qm4.cfs, _qm4.si, _qm5.cfe, _qm5.cfs, _qm5.si, _qm6.cfe, _qm6.cfs, _qm6.si, _qm7.cfe, _qm7.cfs, _qm7.si, _qm8.cfe, _qm8.cfs, _qm8.si, _qm9.cfe, _qm9.cfs, _qm9.si, _qma.cfe, _qma.cfs, _qma.si, _qmb.cfe, _qmb.cfs, _qmb.si, _qmc.cfe, _qmc.cfs, _qmc.si, _qmd.cfe, _qmd.cfs, _qmd.si, _qme.cfe, _qme.cfs, _qme.si, _qmf.cfe, _qmf.cfs, _qmf.si, _qmg.cfe, _qmg.cfs, _qmg.si, _qmh.cfe, _qmh.cfs, _qmh.si, _qmi.cfe, _qmi.cfs, _qmi.si, _qmj.cfe, _qmj.cfs, _qmj.si, _qmk.cfe, _qmk.cfs, _qmk.si, _qml.cfe, _qml.cfs, _qml.si, _qmm.cfe, _qmm.cfs, _qmm.si, _qmn.cfe, _qmn.cfs, _qmn.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1422237634958,generation=2,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qlv.cfe, _qlv.cfs, _qlv.si, _qlw.cfe, _qlw.cfs, _qlw.si, _qlx.cfe, _qlx.cfs, _qlx.si, _qly.cfe, _qly.cfs, _qly.si, _qlz.cfe, _qlz.cfs, _qlz.si, _qm0.cfe, _qm0.cfs, _qm0.si, _qm1.cfe, _qm1.cfs, _qm1.si, _qm2.cfe, _qm2.cfs, _qm2.si, _qm3.cfe, _qm3.cfs, _qm3.si, _qm4.cfe, _qm4.cfs, _qm4.si, _qm5.cfe, _qm5.cfs, _qm5.si, _qm6.cfe, _qm6.cfs, _qm6.si, _qm7.cfe, _qm7.cfs, _qm7.si, _qm8.cfe, _qm8.cfs, _qm8.si, _qm9.cfe, _qm9.cfs, _qm9.si, _qma.cfe, _qma.cfs, _qma.si, _qmb.cfe, _qmb.cfs, _qmb.si, _qmc.cfe, _qmc.cfs, _qmc.si, _qmd.cfe, _qmd.cfs, _qmd.si, _qme.cfe, _qme.cfs, _qme.si, _qmf.cfe, _qmf.cfs, _qmf.si, _qmg.cfe, _qmg.cfs, _qmg.si, _qmh.cfe, _qmh.cfs, _qmh.si, _qmi.cfe, _qmi.cfs, _qmi.si, _qmj.cfe, _qmj.cfs, _qmj.si, _qmk.cfe, _qmk.cfs, _qmk.si, _qml.cfe, _qml.cfs, _qml.si, _qmm.cfe, _qmm.cfs, _qmm.si, _qmn.cfe, _qmn.cfs, _qmn.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_2]}]> but was:<[{indexVersion=1422237634958,generation=3,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmp.cfe, _qmp.cfs, _qmp.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_3]}, {indexVersion=1422237634958,generation=2,filelist=[_ak9.cfe, _ak9.cfs, _ak9.si, _l1a.cfe, _l1a.cfs, _l1a.si, _qlv.cfe, _qlv.cfs, _qlv.si, _qlw.cfe, _qlw.cfs, _qlw.si, _qlx.cfe, _qlx.cfs, _qlx.si, _qly.cfe, _qly.cfs, _qly.si, _qlz.cfe, _qlz.cfs, _qlz.si, _qm0.cfe, _qm0.cfs, _qm0.si, _qm1.cfe, _qm1.cfs, _qm1.si, _qm2.cfe, _qm2.cfs, _qm2.si, _qm3.cfe, _qm3.cfs, _qm3.si, _qm4.cfe, _qm4.cfs, _qm4.si, _qm5.cfe, _qm5.cfs, _qm5.si, _qm6.cfe, _qm6.cfs, _qm6.si, _qm7.cfe, _qm7.cfs, _qm7.si, _qm8.cfe, _qm8.cfs, _qm8.si, _qm9.cfe, _qm9.cfs, _qm9.si, _qma.cfe, _qma.cfs, _qma.si, _qmb.cfe, _qmb.cfs, _qmb.si, _qmc.cfe, _qmc.cfs, _qmc.si, _qmd.cfe, _qmd.cfs, _qmd.si, _qme.cfe, _qme.cfs, _qme.si, _qmf.cfe, _qmf.cfs, _qmf.si, _qmg.cfe, _qmg.cfs, _qmg.si, _qmh.cfe, _qmh.cfs, _qmh.si, _qmi.cfe, _qmi.cfs, _qmi.si, _qmj.cfe, _qmj.cfs, _qmj.si, _qmk.cfe, _qmk.cfs, _qmk.si, _qml.cfe, _qml.cfs, _qml.si, _qmm.cfe, _qmm.cfs, _qmm.si, _qmn.cfe, _qmn.cfs, _qmn.si, _qmo.cfe, _qmo.cfs, _qmo.si, _qmq.cfe, _qmq.cfs, _qmq.si, _qmr.cfe, _qmr.cfs, _qmr.si, _qms.cfe, _qms.cfs, _qms.si, _qmt.cfe, _qmt.cfs, _qmt.si, _qmu.cfe, _qmu.cfs, _qmu.si, _qmv.cfe, _qmv.cfs, _qmv.si, _qmw.cfe, _qmw.cfs, _qmw.si, _qmx.cfe, _qmx.cfs, _qmx.si, _qmy.cfe, _qmy.cfs, _qmy.si, _qmz.cfe, _qmz.cfs, _qmz.si, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:642FE292E21C0FAD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53147/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53147/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53147/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53124/c8n_1x3_lf_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53124/c8n_1x3_lf_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53124/c8n_1x3_lf_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:47240/kq_bfp/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47240/kq_bfp/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:175)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:153)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:133)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62655/rt_/jw/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62655/rt_/jw/collection1
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:123)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:109)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62655/rt_/jw/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42383/vgim/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42383/vgim/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([41F8F9A2925401AE:C9ACC6783CA86C56]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:299)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:213)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42383/vgim/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9707 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/init-core-data-001
   [junit4]   2> 453383 T3074 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /vgim/
   [junit4]   2> 453390 T3074 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 453391 T3075 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 453491 T3074 oasc.ZkTestServer.run start zk server on port:42367
   [junit4]   2> 453492 T3074 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 453493 T3074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 453499 T3082 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c83ba5 name:ZooKeeperConnection Watcher:127.0.0.1:42367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 453499 T3074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 453500 T3074 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 453500 T3074 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 453504 T3074 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 453505 T3074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 453507 T3085 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@147c572b name:ZooKeeperConnection Watcher:127.0.0.1:42367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 453507 T3074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 453507 T3074 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 453508 T3074 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 453510 T3074 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 453512 T3074 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 453514 T3074 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 453516 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 453517 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 453521 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 453521 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 453624 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 453625 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 453627 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 453627 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 453629 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 453630 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 453632 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 453633 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 453635 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 453635 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 453637 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 453638 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 453640 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 453640 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 453642 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 453643 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 453645 T3074 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 453645 T3074 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 455143 T3074 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores/collection1
   [junit4]   2> 455149 T3074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 455155 T3074 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42384
   [junit4]   2> 455156 T3074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 455156 T3074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 455157 T3074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001
   [junit4]   2> 455158 T3074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/'
   [junit4]   2> 455266 T3074 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/solr.xml
   [junit4]   2> 455364 T3074 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores
   [junit4]   2> 455365 T3074 oasc.CoreContainer.<init> New CoreContainer 919017785
   [junit4]   2> 455365 T3074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/]
   [junit4]   2> 455367 T3074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 455367 T3074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 455367 T3074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 455368 T3074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 455368 T3074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 455368 T3074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 455369 T3074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 455369 T3074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 455369 T3074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 455370 T3074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 455370 T3074 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 455371 T3074 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 455371 T3074 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 455371 T3074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 455372 T3074 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 455372 T3074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42367/solr
   [junit4]   2> 455373 T3074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 455373 T3074 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 455374 T3074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 455376 T3097 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6618b505 name:ZooKeeperConnection Watcher:127.0.0.1:42367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 455377 T3074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 455377 T3074 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 455379 T3074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 455381 T3100 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3be50b63 name:ZooKeeperConnection Watcher:127.0.0.1:42367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 455382 T3074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 455383 T3074 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 455386 T3074 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 455388 T3074 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 455390 T3074 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 455393 T3074 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 455395 T3074 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 455396 T3074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42383_vgim
   [junit4]   2> 455397 T3074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42383_vgim
   [junit4]   2> 455399 T3074 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 455401 T3074 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 455402 T3074 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 455404 T3074 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42383_vgim
   [junit4]   2> 455404 T3074 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 455406 T3074 oasc.Overseer.start Overseer (id=93207621800689667-127.0.0.1:42383_vgim-n_0000000000) starting
   [junit4]   2> 455408 T3074 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 455415 T3074 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 455416 T3102 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 455416 T3074 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 455419 T3074 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 455420 T3074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 455423 T3101 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 455424 T3074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores
   [junit4]   2> 455426 T3074 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores/collection1/
   [junit4]   2> 455427 T3074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 455428 T3104 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 455428 T3104 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 455429 T3104 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 455429 T3100 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 455431 T3101 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42383/vgim",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42383_vgim",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 455432 T3101 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42383/vgim",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42383_vgim",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 455432 T3101 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 455433 T3101 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 455434 T3100 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 456430 T3104 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 456430 T3104 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 456431 T3104 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 456431 T3104 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 456432 T3104 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 456432 T3104 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores/collection1/'
   [junit4]   2> 456448 T3104 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 456451 T3104 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 456462 T3104 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 456485 T3104 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 456543 T3104 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 456644 T3104 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 456655 T3104 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 457036 T3104 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 457054 T3104 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 457059 T3104 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 457073 T3104 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 457078 T3104 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 457082 T3104 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 457083 T3104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 457084 T3104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 457084 T3104 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 457085 T3104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 457085 T3104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 457085 T3104 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 457085 T3104 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 457086 T3104 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/tempDir-001/control/data/
   [junit4]   2> 457086 T3104 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4c85b97c
   [junit4]   2> 457088 T3104 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/tempDir-001/control/data
   [junit4]   2> 457089 T3104 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/tempDir-001/control/data/index/
   [junit4]   2> 457089 T3104 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 457090 T3104 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/tempDir-001/control/data/index
   [junit4]   2> 457090 T3104 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 457091 T3104 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52fe2c8d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d1f078c),segFN=segments_1,generation=1}
   [junit4]   2> 457091 T3104 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 457096 T3104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 457096 T3104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 457096 T3104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 457097 T3104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 457097 T3104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 457097 T3104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 457098 T3104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 457098 T3104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 457098 T3104 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 457099 T3104 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 457100 T3104 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 457100 T3104 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 457101 T3104 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 457101 T3104 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 457102 T3104 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 457102 T3104 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 457102 T3104 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 457103 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 457104 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 457104 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 457105 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 457105 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 457106 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 457106 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 457107 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 457107 T3104 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 457108 T3104 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 457126 T3104 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 457128 T3104 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 457130 T3104 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 457132 T3104 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 457134 T3104 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 457138 T3104 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 457138 T3104 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 457139 T3104 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2115844357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 457140 T3104 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52fe2c8d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d1f078c),segFN=segments_1,generation=1}
   [junit4]   2> 457140 T3104 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 457141 T3104 oass.SolrIndexSearcher.<init> Opening Searcher@45f55a56[collection1] main
   [junit4]   2> 457141 T3104 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 457142 T3104 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 457142 T3104 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 457143 T3104 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 457143 T3104 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 457143 T3104 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 457144 T3104 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 457144 T3104 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 457144 T3104 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 457144 T3104 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 457145 T3105 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@45f55a56[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 457146 T3104 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 457147 T3108 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42383/vgim collection:control_collection shard:shard1
   [junit4]   2> 457147 T3074 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 457148 T3074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 457148 T3108 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 457152 T3074 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 457153 T3108 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 457153 T3074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 457155 T3112 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d9a3f2e name:ZooKeeperConnection Watcher:127.0.0.1:42367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 457155 T3074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 457155 T3100 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 457155 T3074 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 457156 T3108 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 457156 T3108 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 457156 T3101 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 457156 T3074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C386 name=collection1 org.apache.solr.core.SolrCore@49950c0d url=http://127.0.0.1:42383/vgim/collection1 node=127.0.0.1:42383_vgim C386_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:42383/vgim, state=down, node_name=127.0.0.1:42383_vgim}
   [junit4]   2> 457156 T3108 C386 P42383 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42383/vgim/collection1/
   [junit4]   2> 457157 T3108 C386 P42383 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 457158 T3108 C386 P42383 oasc.SyncStrategy.syncToMe http://127.0.0.1:42383/vgim/collection1/ has no replicas
   [junit4]   2> 457158 T3100 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 457158 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 457158 T3108 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42383/vgim/collection1/ shard1
   [junit4]   2> 457160 T3108 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 457161 T3074 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 457164 T3113 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 457165 T3101 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42383/vgim",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 457268 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 457317 T3108 oasc.ZkController.register We are http://127.0.0.1:42383/vgim/collection1/ and leader is http://127.0.0.1:42383/vgim/collection1/
   [junit4]   2> 457318 T3108 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42383/vgim
   [junit4]   2> 457318 T3108 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 457318 T3108 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 457318 T3108 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 457320 T3100 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 457322 T3101 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42383/vgim",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:42383_vgim",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 457322 T3101 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42383/vgim",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:42383_vgim",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 457426 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 457426 T3100 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 458343 T3074 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 41F8F9A2925401AE-001/shard-1-001/cores/collection1
   [junit4]   2> 458345 T3074 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 458346 T3074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 458349 T3074 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42391
   [junit4]   2> 458349 T3074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 458350 T3074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 458350 T3074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cl

[...truncated too long message...]

ecute [collection1] webapp=/solr path=/replication params={_trace=getDetails&version=2&qt=/replication&command=details&wt=javabin} status=0 QTime=3 
   [junit4]   2> 3081906 T6788 oas.SolrTestCaseJ4.tearDown ###Ending doTestSnapPullWithMasterUrl
   [junit4]   2> 3081907 T6788 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=687228844
   [junit4]   2> 3081909 T6788 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@c5e9832
   [junit4]   2> 3081909 T6788 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=497,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 3081911 T6788 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 3081911 T6788 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 3081911 T6788 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 3081913 T6788 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 3081914 T6788 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 3081914 T6788 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 3081915 T6788 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data
   [junit4]   2> 3081915 T6788 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data/index.20150125180748145 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data/index.20150125180748145;done=false>>]
   [junit4]   2> 3081916 T6788 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-001/collection1/data/index.20150125180748145
   [junit4]   2> 3081949 T6788 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3082111 T6788 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1038532576
   [junit4]   2> 3082113 T6788 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1f874c79
   [junit4]   2> 3082113 T6788 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 3082114 T6788 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 3082114 T6788 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 3082114 T6788 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 3082116 T6788 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 3082116 T6788 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 3082117 T6788 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 3082117 T6788 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data
   [junit4]   2> 3082118 T6788 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data/index.20150125180747032 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data/index.20150125180747032;done=false>>]
   [junit4]   2> 3082118 T6788 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001/solr-instance-002/collection1/data/index.20150125180747032
   [junit4]   2> 3082151 T6788 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3082364 T6788 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.TestReplicationHandler 41F8F9A2925401AE-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=pl, timezone=Pacific/Pitcairn
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=162748632,total=374341632
   [junit4]   2> NOTE: All tests run in this JVM: [TestDistributedGrouping, ExternalCollectionsTest, TestMiniSolrCloudClusterSSL, TestSolrConfigHandlerCloud, DocValuesMissingTest, TestQueryTypes, TestMissingGroups, TestFieldResource, OverriddenZkACLAndCredentialsProvidersTest, RemoteQueryErrorTest, TestReRankQParserPlugin, TestIndexingPerformance, TestManagedSchema, MultiThreadedOCPTest, TestUtils, SampleTest, TestArbitraryIndexDir, NumericFieldsTest, TestDynamicFieldCollectionResource, TestShortCircuitedRequests, TestSchemaVersionResource, ClusterStateTest, TestImplicitCoreProperties, BadIndexSchemaTest, TestBlendedInfixSuggestions, TestRangeQuery, TermVectorComponentDistributedTest, ReplicationFactorTest, TestJoin, TestBulkSchemaConcurrent, InfoHandlerTest, DocValuesMultiTest, TestConfigOverlay, DeleteLastCustomShardedReplicaTest, HdfsDirectoryFactoryTest, SolrCoreTest, FileUtilsTest, TestFiltering, CSVRequestHandlerTest, TestCopyFieldCollectionResource, TestSolrConfigHandlerConcurrent, BufferStoreTest, MultiTermTest, CursorMarkTest, IndexSchemaRuntimeFieldTest, ResponseLogComponentTest, TestDocBasedVersionConstraints, LeaderInitiatedRecoveryOnCommitTest, JsonLoaderTest, HighlighterMaxOffsetTest, ScriptEngineTest, TestSystemIdResolver, TestCloudInspectUtil, UpdateRequestProcessorFactoryTest, TestCloudManagedSchema, DocExpirationUpdateProcessorFactoryTest, CurrencyFieldXmlFileTest, TestOverriddenPrefixQueryForCustomFieldType, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, OverseerTest, TestReplicationHandler]
   [junit4] Completed on J3 in 1745.87s, 13 tests, 1 failure <<< FAILURES!

[...truncated 18 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1846 tests, 5 errors, 2 failures, 55 ignored (8 assumptions)

Total time: 160 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 708374921 bytes
Compression is 0.0%
Took 2 min 3 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 743 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/743/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36081/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36081/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:189)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36081/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30917/wvr/b/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30917/wvr/b/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:468)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30917/wvr/b/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:36409/pmuru/u/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36409/pmuru/u/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:175)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:153)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:133)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61027/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61027/collection1
	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:123)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:109)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61027/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61018/tw/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61018/tw/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:299)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:213)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61018/tw/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more




Build Log:
[...truncated 9806 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/init-core-data-001
   [junit4]   2> 227364 T721 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 227369 T721 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 227370 T722 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 227470 T721 oasc.ZkTestServer.run start zk server on port:36072
   [junit4]   2> 227471 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 227471 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 227475 T729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7af7712e name:ZooKeeperConnection Watcher:127.0.0.1:36072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 227476 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 227476 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 227476 T721 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 227480 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 227480 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 227482 T732 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61b0f345 name:ZooKeeperConnection Watcher:127.0.0.1:36072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 227482 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 227483 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 227483 T721 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 227486 T721 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 227489 T721 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 227491 T721 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 227493 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 227494 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 227498 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 227499 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 227502 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 227502 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 227505 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 227505 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 227508 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 227508 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 227511 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 227511 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 227514 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 227515 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 227518 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 227518 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 227521 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 227522 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 227524 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 227525 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 227527 T721 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 227528 T721 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 229029 T721 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores/collection1
   [junit4]   2> 229033 T721 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 229037 T721 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36082
   [junit4]   2> 229038 T721 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 229038 T721 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 229038 T721 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001
   [junit4]   2> 229039 T721 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/'
   [junit4]   2> 229081 T721 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/solr.xml
   [junit4]   2> 229133 T721 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores
   [junit4]   2> 229133 T721 oasc.CoreContainer.<init> New CoreContainer 148003121
   [junit4]   2> 229133 T721 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/]
   [junit4]   2> 229134 T721 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 229134 T721 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 229135 T721 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 229135 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 229135 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 229135 T721 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 229135 T721 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 229136 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 229136 T721 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 229136 T721 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 229136 T721 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 229137 T721 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 229137 T721 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 229137 T721 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 229137 T721 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 229138 T721 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36072/solr
   [junit4]   2> 229138 T721 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 229138 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 229139 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229141 T744 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@307221e8 name:ZooKeeperConnection Watcher:127.0.0.1:36072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229141 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 229142 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 229143 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229145 T747 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@269162d5 name:ZooKeeperConnection Watcher:127.0.0.1:36072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229145 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 229146 T721 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 229149 T721 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 229152 T721 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 229154 T721 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 229157 T721 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 229160 T721 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 229161 T721 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36081_
   [junit4]   2> 229162 T721 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36081_
   [junit4]   2> 229164 T721 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 229166 T721 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 229168 T721 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 229170 T721 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:36081_
   [junit4]   2> 229170 T721 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 229172 T721 oasc.Overseer.start Overseer (id=93199761829265411-127.0.0.1:36081_-n_0000000000) starting
   [junit4]   2> 229174 T721 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 229182 T721 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 229183 T749 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 229183 T721 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 229185 T721 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 229187 T721 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 229190 T748 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 229191 T721 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores
   [junit4]   2> 229193 T721 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores/collection1/
   [junit4]   2> 229193 T721 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 229194 T751 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 229194 T751 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 229195 T747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229195 T751 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 229198 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36081",
   [junit4]   2> 	  "node_name":"127.0.0.1:36081_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 229199 T748 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36081",
   [junit4]   2> 	  "node_name":"127.0.0.1:36081_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 229199 T748 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 229200 T748 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 229201 T747 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230195 T751 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 230196 T751 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 230197 T751 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 230197 T751 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 230198 T751 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 230198 T751 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores/collection1/'
   [junit4]   2> 230216 T751 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 230219 T751 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 230231 T751 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 230253 T751 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 230300 T751 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 230302 T751 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 230313 T751 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 230679 T751 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 230691 T751 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 230694 T751 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 230712 T751 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230717 T751 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230722 T751 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 230723 T751 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 230723 T751 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 230724 T751 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 230725 T751 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 230725 T751 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 230725 T751 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 230726 T751 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 230726 T751 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/control/data/
   [junit4]   2> 230726 T751 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@40c8c039
   [junit4]   2> 230727 T751 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/control/data
   [junit4]   2> 230727 T751 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/control/data/index/
   [junit4]   2> 230728 T751 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 230728 T751 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/control/data/index
   [junit4]   2> 230728 T751 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.365234375, noCFSRatio=0.48675146056135443]
   [junit4]   2> 230729 T751 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6e0f2ee3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26cf7188; maxCacheMB=0.0888671875 maxMergeSizeMB=0.890625)),segFN=segments_1,generation=1}
   [junit4]   2> 230730 T751 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230735 T751 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 230736 T751 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 230736 T751 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 230736 T751 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230736 T751 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230737 T751 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 230737 T751 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230737 T751 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230738 T751 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 230738 T751 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 230739 T751 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 230739 T751 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 230740 T751 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 230740 T751 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 230741 T751 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 230741 T751 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 230742 T751 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 230742 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 230744 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 230744 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 230745 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 230745 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 230746 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 230746 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 230747 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 230747 T751 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 230748 T751 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 230766 T751 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230768 T751 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230770 T751 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230772 T751 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230774 T751 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 230778 T751 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 230778 T751 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 230779 T751 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7d2f3ad3
   [junit4]   2> 230779 T751 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6e0f2ee3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26cf7188; maxCacheMB=0.0888671875 maxMergeSizeMB=0.890625)),segFN=segments_1,generation=1}
   [junit4]   2> 230780 T751 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230780 T751 oass.SolrIndexSearcher.<init> Opening Searcher@5f1b01f4[collection1] main
   [junit4]   2> 230780 T751 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 230782 T751 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 230782 T751 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 230783 T751 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 230783 T751 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 230783 T751 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 230784 T751 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 230784 T751 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 230784 T751 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 230785 T751 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 230786 T752 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5f1b01f4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 230786 T751 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 230787 T755 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36081 collection:control_collection shard:shard1
   [junit4]   2> 230788 T721 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 230788 T721 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 230788 T755 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 230792 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230793 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230795 T759 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3addcc8a name:ZooKeeperConnection Watcher:127.0.0.1:36072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230795 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230795 T755 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 230795 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230797 T721 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 230798 T747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230798 T755 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 230798 T755 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C254 name=collection1 org.apache.solr.core.SolrCore@2301eff0 url=http://127.0.0.1:36081/collection1 node=127.0.0.1:36081_ C254_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:36081, node_name=127.0.0.1:36081_}
   [junit4]   2> 230799 T755 C254 P36081 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36081/collection1/
   [junit4]   2> 230799 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 230799 T755 C254 P36081 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 230799 T755 C254 P36081 oasc.SyncStrategy.syncToMe http://127.0.0.1:36081/collection1/ has no replicas
   [junit4]   2> 230800 T755 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36081/collection1/ shard1
   [junit4]   2> 230800 T755 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 230801 T759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230801 T747 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230801 T721 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 230802 T721 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 230802 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230803 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230805 T762 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3861ec23 name:ZooKeeperConnection Watcher:127.0.0.1:36072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230805 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230806 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230808 T763 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230809 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36081",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 230813 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 230813 T748 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 230814 T759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230818 T763 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230819 T748 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 230821 T759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230859 T755 oasc.ZkController.register We are http://127.0.0.1:36081/collection1/ and leader is http://127.0.0.1:36081/collection1/
   [junit4]   2> 230860 T755 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36081
   [junit4]   2> 230860 T755 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 230860 T755 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 230860 T755 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 230862 T747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230864 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36081",
   [junit4]   2> 	  "node_name":"127.0.0.1:36081_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 230865 T748 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36081",
   [junit4]   2> 	  "node_name":"127.0.0.1:36081_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 230969 T747 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230970 T759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 232321 T721 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores/collection1
   [junit4]   2> 232325 T721 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 232326 T721 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 232328 T721 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36097
   [junit4]   2> 232329 T721 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 232329 T721 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 232330 T721 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001
   [junit4]   2> 232330 T721 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/'
   [junit4]   2> 232374 T721 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/solr.xml
   [junit4]   2> 232430 T721 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores
   [junit4]   2> 232431 T721 oasc.CoreContainer.<init> New CoreContainer 636207173
   [junit4]   2> 232431 T721 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/]
   [junit4]   2> 232433 T721 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 232433 T721 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 232433 T721 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 232434 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 232434 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 232434 T721 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 232435 T721 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 232435 T721 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 232435 T721 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 232436 T721 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 232436 T721 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 232437 T721 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 232437 T721 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 232437 T721 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 232438 T721 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 232438 T721 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36072/solr
   [junit4]   2> 232438 T721 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 232439 T721 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 232440 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 232442 T775 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@479c124c name:ZooKeeperConnection Watcher:127.0.0.1:36072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 232442 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 232443 T721 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 232445 T721 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 232447 T778 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30081fcc name:ZooKeeperConnection Watcher:127.0.0.1:36072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 232447 T721 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 232455 T721 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 233461 T721 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36096_
   [junit4]   2> 233463 T721 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36096_
   [junit4]   2> 233467 T721 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 233471 T721 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores
   [junit4]   2> 233473 T721 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores/collection1/
   [junit4]   2> 233473 T721 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 233477 T779 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 233477 T779 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233479 T747 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233480 T779 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 233480 T779 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 233480 T748 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36096",
   [junit4]   2> 	  "node_name":"127.0.0.1:36096_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 233481 T779 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 233481 T748 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36096",
   [junit4]   2> 	  "node_name":"127.0.0.1:36096_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 233482 T748 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 233482 T748 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 233483 T779 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 233483 T779 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 233484 T748 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 233485 T778 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 233486 T778 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 234483 T779 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 234484 T779 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 234485 T779 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 234485 T779 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 234486 T779 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 234486 T779 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores/collection1/'
   [junit4]   2> 234503 T779 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 234506 T779 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 234518 T779 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 234540 T779 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 234587 T779 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 234589 T779 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 234600 T779 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 234976 T779 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 234987 T779 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 234990 T779 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 235008 T779 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235013 T779 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235017 T779 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235018 T779 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235019 T779 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235019 T779 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235020 T779 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235021 T779 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235021 T779 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 235021 T779 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 235022 T779 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/jetty1/
   [junit4]   2> 235022 T779 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@40c8c039
   [junit4]   2> 235023 T779 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/jetty1
   [junit4]   2> 235023 T779 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/jetty1/index/
   [junit4]   2> 235023 T779 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 235024 T779 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.HttpPartitionTest C5E54D574BE991D0-001/tempDir-001/jetty1/index
   [junit4]   2> 235024 T779 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.365234375, noCFSRatio=0.48675146056135443]
   [junit4]   2> 235025 T779 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@2055483d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3ef9ad60; maxCacheMB=0.0888671875 maxMergeSizeMB=0.890625)),segFN=segments_1,generation=1}
   [junit4]   2> 235025 T779 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 235032 T779 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 235033 T779 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 235033 T779 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 235033 T779 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 235034 T779 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 235034 T779 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 235034 T779 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-imp

[...truncated too long message...]

RROR   75.0s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61018/tw/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C5E54D574BE991D0:4DB1728DE515FC28]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:575)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:787)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:299)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:213)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:110)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61018/tw/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:572)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 2164225 T6107 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest C5E54D574BE991D0-001
   [junit4]   2> 75009 T6106 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=Memory doPackFST= false), id=Lucene50(blocksize=128), intDefault=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=no_NO, timezone=America/Barbados
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=301691312,total=434634752
   [junit4]   2> NOTE: All tests run in this JVM: [TestExpandComponent, TestStressRecovery, TestSolr4Spatial, DocumentAnalysisRequestHandlerTest, CloudExitableDirectoryReaderTest, TestFuzzyAnalyzedSuggestions, HighlighterTest, SchemaVersionSpecificBehaviorTest, SuggesterFSTTest, TestSerializedLuceneMatchVersion, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestStressVersions, TestRawResponseWriter, TestRecoveryHdfs, AutoCommitTest, SolrXmlInZkTest, DirectSolrConnectionTest, TestCoreDiscovery, TestLMJelinekMercerSimilarityFactory, DistributedTermsComponentTest, TestFunctionQuery, SimplePostToolTest, TestInitQParser, HttpPartitionTest, TestClusterStateMutator, HdfsSyncSliceTest, FieldMutatingUpdateProcessorTest, FieldAnalysisRequestHandlerTest, DistributedFacetPivotLargeTest, TestDistributedMissingSort, TermVectorComponentTest, TestExactStatsCache, LoggingHandlerTest, CollectionsAPIAsyncDistributedZkTest, TestDefaultSearchFieldResource, OverseerStatusTest, ConnectionManagerTest, TestInfoStreamLogging, AnalyticsQueryTest, JSONWriterTest, TestSortingResponseWriter, PreAnalyzedFieldTest, BJQParserTest, PathHierarchyTokenizerFactoryTest, CircularListTest, TestBlobHandler, OpenCloseCoreStressTest, DisMaxRequestHandlerTest, SortByFunctionTest, TestConfigReload, TestSolrXml, AliasIntegrationTest, SuggesterTSTTest, TestZkChroot, ShardSplitTest, TestAddFieldRealTimeGet, FullSolrCloudDistribCmdsTest, ZkSolrClientTest, TestDistributedGrouping, TestFaceting, TestHashPartitioner, ZkControllerTest, TestJoin, HardAutoCommitTest, TestRangeQuery, SolrCmdDistributorTest, QueryElevationComponentTest, ConvertedLegacyTest, TestFiltering, DirectUpdateHandlerTest, TestIndexSearcher, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, TestExtendedDismaxParser, DocValuesTest, TestStressLucene, TestTrie, WordBreakSolrSpellCheckerTest, TestUpdate, DefaultValueUpdateProcessorTest, DistanceFunctionTest, SolrInfoMBeanTest, DebugComponentTest, TestQueryUtils, DirectSolrSpellCheckerTest, PrimitiveFieldTypeTest, FileBasedSpellCheckerTest, TestValueSourceCache, TestSolrQueryParser, TestArbitraryIndexDir, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestJmxIntegration, MBeansHandlerTest, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestComponentsName, TestLFUCache, AlternateDirectoryTest, TestQuerySenderNoQuery, SolrIndexConfigTest, TestSolrDeletionPolicy2, NumericFieldsTest, MinimalSchemaTest, TestConfig, TestPhraseSuggestions, EchoParamsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestFastLRUCache, DateMathParserTest, TestSystemIdResolver, SpellingQueryConverterTest, TestSolrJ, TestDocumentBuilder, SystemInfoHandlerTest, UUIDFieldTest, CursorPagingTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistributedQueueTest, LeaderFailoverAfterPartitionTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 75.80s, 1 test, 1 error <<< FAILURES!

[...truncated 301 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1846 tests, 5 errors, 55 ignored (8 assumptions)

Total time: 157 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 724687213 bytes
Compression is 0.0%
Took 2 min 1 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 742 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/742/

5 tests failed.
FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:117)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:103)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.doTest(RecoveryAfterSoftCommitTest.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.GeneratedMethodAccessor59.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:35856/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:35856/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.GeneratedMethodAccessor59.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 47 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9784 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/init-core-data-001
   [junit4]   2> 208949 T336 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 208956 T336 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 208989 T336 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 208998 T337 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 209302 T336 oasc.ZkTestServer.run start zk server on port:62198
   [junit4]   2> 209317 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 209641 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 209862 T344 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14503606 name:ZooKeeperConnection Watcher:127.0.0.1:62198 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 209863 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 209864 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 209878 T336 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 210026 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 210027 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 210029 T347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@662cf9fd name:ZooKeeperConnection Watcher:127.0.0.1:62198/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 210030 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 210030 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 210031 T336 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 210035 T336 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 210039 T336 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 210042 T336 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 210046 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 210047 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 210053 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 210054 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 210059 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210059 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210064 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 210065 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 210069 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 210070 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 210074 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 210074 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 210079 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 210079 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 210083 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210084 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210088 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210089 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210093 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 210093 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 210098 T336 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 210098 T336 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 212210 T336 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores/collection1
   [junit4]   2> 212215 T336 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 212218 T336 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62209
   [junit4]   2> 212219 T336 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 212219 T336 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 212220 T336 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001
   [junit4]   2> 212220 T336 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/'
   [junit4]   2> 212276 T336 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/solr.xml
   [junit4]   2> 212360 T336 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores
   [junit4]   2> 212360 T336 oasc.CoreContainer.<init> New CoreContainer 809524253
   [junit4]   2> 212360 T336 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/]
   [junit4]   2> 212362 T336 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 212362 T336 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 212362 T336 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 212363 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 212363 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 212363 T336 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 212364 T336 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 212364 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 212364 T336 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 212364 T336 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 212365 T336 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 212365 T336 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 212366 T336 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 212366 T336 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 212366 T336 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 212367 T336 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62198/solr
   [junit4]   2> 212436 T336 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 212436 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212437 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212440 T359 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cd52850 name:ZooKeeperConnection Watcher:127.0.0.1:62198 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212440 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212441 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212468 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212470 T362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d2647f0 name:ZooKeeperConnection Watcher:127.0.0.1:62198/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212471 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212500 T336 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 212522 T336 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 212533 T336 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 212537 T336 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 212542 T336 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 212555 T336 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 212557 T336 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62208_
   [junit4]   2> 212567 T336 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62208_
   [junit4]   2> 212572 T336 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 212575 T336 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 212578 T336 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 212605 T336 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62208_
   [junit4]   2> 212605 T336 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 212608 T336 oasc.Overseer.start Overseer (id=93193985414332419-127.0.0.1:62208_-n_0000000000) starting
   [junit4]   2> 212635 T336 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 212666 T336 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 212695 T364 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 212695 T336 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 212698 T336 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 212701 T336 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212732 T363 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 212733 T336 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores
   [junit4]   2> 212758 T336 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores/collection1/
   [junit4]   2> 212759 T336 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 212760 T366 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 212760 T366 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 212783 T362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212783 T366 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 212794 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:62208_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62208",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 212855 T363 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:62208_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62208",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 212863 T363 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 212901 T363 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 212922 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213784 T366 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 213784 T366 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 213785 T366 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 213785 T366 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 213787 T366 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 213788 T366 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores/collection1/'
   [junit4]   2> 213837 T366 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 213862 T366 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 213895 T366 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 213923 T366 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 213971 T366 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 213973 T366 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 213983 T366 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 214374 T366 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 214387 T366 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 214390 T366 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 214408 T366 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214414 T366 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214419 T366 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214420 T366 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214421 T366 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214421 T366 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214423 T366 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214423 T366 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214423 T366 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 214423 T366 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 214424 T366 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/control/data/
   [junit4]   2> 214424 T366 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bdb8813
   [junit4]   2> 214441 T366 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/control/data
   [junit4]   2> 214441 T366 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/control/data/index/
   [junit4]   2> 214441 T366 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 214442 T366 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/control/data/index
   [junit4]   2> 214442 T366 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@20b442ae
   [junit4]   2> 214443 T366 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1a85be48 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@428f7692),segFN=segments_1,generation=1}
   [junit4]   2> 214443 T366 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214453 T366 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 214453 T366 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 214454 T366 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 214454 T366 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214454 T366 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214455 T366 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 214455 T366 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214455 T366 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214456 T366 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 214465 T366 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 214466 T366 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 214467 T366 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 214468 T366 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 214469 T366 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 214470 T366 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 214471 T366 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 214472 T366 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 214474 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 214476 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 214477 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 214478 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 214479 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 214480 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 214481 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 214482 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 214483 T366 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 214484 T366 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 214506 T366 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214508 T366 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214510 T366 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214512 T366 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214514 T366 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 214518 T366 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 214518 T366 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 214520 T366 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=23, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 214521 T366 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1a85be48 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@428f7692),segFN=segments_1,generation=1}
   [junit4]   2> 214521 T366 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214521 T366 oass.SolrIndexSearcher.<init> Opening Searcher@3b1fab8a[collection1] main
   [junit4]   2> 214522 T366 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 214523 T366 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 214524 T366 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 214537 T366 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 214537 T366 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 214537 T366 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214538 T366 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 214538 T366 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214539 T366 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 214539 T366 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 214542 T367 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3b1fab8a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 214549 T366 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 214550 T370 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62208 collection:control_collection shard:shard1
   [junit4]   2> 214550 T336 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 214551 T336 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 214585 T370 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 214595 T370 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 214599 T362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214600 T370 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 214600 T370 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 214600 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C312 name=collection1 org.apache.solr.core.SolrCore@5f2ff1eb url=http://127.0.0.1:62208/collection1 node=127.0.0.1:62208_ C312_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:62208_, state=down, base_url=http://127.0.0.1:62208, core=collection1}
   [junit4]   2> 214610 T370 C312 P62208 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62208/collection1/
   [junit4]   2> 214611 T370 C312 P62208 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 214611 T370 C312 P62208 oasc.SyncStrategy.syncToMe http://127.0.0.1:62208/collection1/ has no replicas
   [junit4]   2> 214611 T370 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62208/collection1/ shard1
   [junit4]   2> 214615 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214619 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214620 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214622 T374 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a0f12fa name:ZooKeeperConnection Watcher:127.0.0.1:62198/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214623 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214623 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214625 T336 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 214629 T370 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 214633 T336 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 214634 T336 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 214634 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214635 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214637 T377 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e7ac37f name:ZooKeeperConnection Watcher:127.0.0.1:62198/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214637 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214637 T378 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214638 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214640 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62208",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 214645 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 214645 T363 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 214647 T374 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214653 T378 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214655 T363 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 214657 T374 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214690 T370 oasc.ZkController.register We are http://127.0.0.1:62208/collection1/ and leader is http://127.0.0.1:62208/collection1/
   [junit4]   2> 214691 T370 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62208
   [junit4]   2> 214691 T370 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 214691 T370 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 214692 T370 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 214694 T362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214698 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:62208_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62208",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2"} current state version: 4
   [junit4]   2> 214699 T363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:62208_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62208",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 214807 T374 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214807 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 216696 T336 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores/collection1
   [junit4]   2> 216700 T336 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 216701 T336 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 216704 T336 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62223
   [junit4]   2> 216705 T336 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 216705 T336 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 216705 T336 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001
   [junit4]   2> 216706 T336 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/'
   [junit4]   2> 216761 T336 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/solr.xml
   [junit4]   2> 216829 T336 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores
   [junit4]   2> 216830 T336 oasc.CoreContainer.<init> New CoreContainer 1152562712
   [junit4]   2> 216830 T336 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/]
   [junit4]   2> 216831 T336 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 216832 T336 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 216832 T336 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 216832 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 216833 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 216833 T336 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 216833 T336 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 216834 T336 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 216834 T336 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 216834 T336 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 216835 T336 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 216835 T336 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 216836 T336 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 216836 T336 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 216836 T336 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 216837 T336 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62198/solr
   [junit4]   2> 216837 T336 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 216837 T336 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 216838 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216841 T390 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ea2e892 name:ZooKeeperConnection Watcher:127.0.0.1:62198 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216841 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216842 T336 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 216845 T336 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216846 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7611f442 name:ZooKeeperConnection Watcher:127.0.0.1:62198/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216847 T336 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216857 T336 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 217867 T336 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62222_
   [junit4]   2> 217869 T336 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62222_
   [junit4]   2> 217875 T336 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 217885 T336 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores
   [junit4]   2> 217888 T336 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores/collection1/
   [junit4]   2> 217888 T336 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 217892 T394 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 217892 T394 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217895 T362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217896 T394 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 217896 T394 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 217896 T363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62222_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62222",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2"} current state version: 5
   [junit4]   2> 217897 T394 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 217897 T363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62222_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62222",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 217898 T363 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 217898 T363 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 217901 T363 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 217906 T394 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 217907 T394 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 217907 T394 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 217907 T394 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 217908 T394 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 217908 T394 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 217909 T394 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 217909 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores/collection1/'
   [junit4]   2> 217927 T394 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 217931 T394 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 217944 T394 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 217967 T394 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 218018 T394 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 218020 T394 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 218030 T394 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 218456 T394 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 218467 T394 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 218470 T394 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 218488 T394 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218493 T394 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218498 T394 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218500 T394 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218501 T394 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218501 T394 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218502 T394 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218503 T394 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218503 T394 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 218503 T394 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 218504 T394 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/jetty1/
   [junit4]   2> 218504 T394 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bdb8813
   [junit4]   2> 218505 T394 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/jetty1
   [junit4]   2> 218505 T394 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/jetty1/index/
   [junit4]   2> 218505 T394 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 218506 T394 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 628FDFDB7F4F4526-001/tempDir-001/jetty1/index
   [junit4]   2> 218506 T394 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@55333d7e
   [junit4]   2> 218507 T394 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@170a46e5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@23b47034),segFN=segments_1,generation=1}
   [junit4]   2> 218507 T394 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 218520 T394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 218520 T394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 218520 T394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 218521 T394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218521 T394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218521 T394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 218522 T394 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218522 T394 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218522 T394 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 218524 T394 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 218526 T394 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 218528 T394 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 218529 T394 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 218531 T394 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 218533 T394 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 218547 T394 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 218548 T394 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 218550 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 218552 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 218554 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 218556 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 218557 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 218559 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 218561 T394 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.Loggin

[...truncated too long message...]

> 1866394 T11962 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1866395 T11961 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36188/
   [junit4]   2> 1866395 T11961 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36170/
   [junit4]   2> 1866396 T11961 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:36176/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=testDistribSearch -Dtests.seed=628FDFDB7F4F4526 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=hi_IN -Dtests.timezone=Africa/Asmera -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.9s J3 | RecoveryAfterSoftCommitTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([628FDFDB7F4F4526:E36951C30810251A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:865)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:730)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:103)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.doTest(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:546)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:543)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1866404 T11961 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 628FDFDB7F4F4526-001
   [junit4]   2> 67890 T11960 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=BlockTreeOrds(blocksize=128), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=hi_IN, timezone=Africa/Asmera
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=190525080,total=506462208
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, EnumFieldTest, TestMissingGroups, TestValueSourceCache, TermVectorComponentDistributedTest, TestReRankQParserPlugin, DeleteLastCustomShardedReplicaTest, DateMathParserTest, CSVRequestHandlerTest, TestCSVLoader, TestShortCircuitedRequests, ShardRoutingTest, TestPostingsSolrHighlighter, SolrCoreTest, TestManagedSchema, TestJmxIntegration, TestCloudPivotFacet, HdfsLockFactoryTest, TestDefaultSimilarityFactory, ResponseLogComponentTest, TestConfigOverlay, TestPseudoReturnFields, BufferStoreTest, OutputWriterTest, TestSolrConfigHandlerCloud, BlockDirectoryTest, RemoteQueryErrorTest, TestHighFrequencyDictionaryFactory, SoftAutoCommitTest, TestExactSharedStatsCache, TestMiniSolrCloudClusterSSL, DefaultValueUpdateProcessorTest, ReplicationFactorTest, InfoHandlerTest, HdfsDirectoryFactoryTest, TestImplicitCoreProperties, MultiTermTest, DocValuesMissingTest, TestFiltering, TestManagedResourceStorage, UpdateRequestProcessorFactoryTest, DocExpirationUpdateProcessorFactoryTest, ClusterStateUpdateTest, XsltUpdateRequestHandlerTest, TestSolrConfigHandlerConcurrent, ScriptEngineTest, TestJoin, FileUtilsTest, CurrencyFieldXmlFileTest, TestCloudInspectUtil, CollectionsAPIDistributedZkTest, SuggesterWFSTTest, TestIndexSearcher, SharedFSAutoReplicaFailoverTest, ExitableDirectoryReaderTest, TestBlendedInfixSuggestions, TestRandomFaceting, TestZkChroot, ZkCLITest, TestDistributedSearch, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestHighlightDedupGrouping, TestTolerantSearch, AsyncMigrateRouteKeyTest, CollectionsAPIAsyncDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest, OverseerStatusTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 68.17s, 1 test, 1 error <<< FAILURES!

[...truncated 284 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1846 tests, 5 errors, 55 ignored (8 assumptions)

Total time: 158 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 725346910 bytes
Compression is 0.0%
Took 2 min 16 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 741 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/741/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:25482: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:906)  at org.apache.solr.cloud.ZkController.register(ZkController.java:810)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /admin/cores. Reason: <pre>    {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:906)  at org.apache.solr.cloud.ZkController.register(ZkController.java:810)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:25482: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:906)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:810)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /admin/cores. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:537)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:906)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:810)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:204)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:228)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:468)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:521)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:488)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:583)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:197)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:186)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:736)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:261)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:142)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:527)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:341)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:460)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:864)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:729)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:117)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:103)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.doTest(RecoveryAfterSoftCommitTest.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:545)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:542)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:864)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:729)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:545)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:542)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:864)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:729)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:545)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:542)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:65090/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65090/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:864)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:729)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:545)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:542)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9729 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/init-core-data-001
   [junit4]   2> 165774 T589 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 165780 T589 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 165781 T589 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 165782 T590 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 165882 T589 oasc.ZkTestServer.run start zk server on port:64362
   [junit4]   2> 165883 T589 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 165884 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 165890 T597 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6401a64d name:ZooKeeperConnection Watcher:127.0.0.1:64362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 165890 T589 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 165891 T589 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 165891 T589 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 165895 T589 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 165896 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 165897 T600 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@765bcbc7 name:ZooKeeperConnection Watcher:127.0.0.1:64362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 165897 T589 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 165898 T589 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 165899 T589 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 165901 T589 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 165903 T589 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 165904 T589 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 165906 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 165907 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 165910 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 165911 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 165914 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 165914 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 165917 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 165917 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 165919 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 165920 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 165922 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 165923 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 165925 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 165926 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 165928 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 165929 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 165931 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 165931 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 165934 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 165934 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 165936 T589 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 165937 T589 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 167703 T589 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores/collection1
   [junit4]   2> 167709 T589 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 167730 T589 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64366
   [junit4]   2> 167731 T589 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 167731 T589 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 167732 T589 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001
   [junit4]   2> 167732 T589 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/'
   [junit4]   2> 167781 T589 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/solr.xml
   [junit4]   2> 167838 T589 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores
   [junit4]   2> 167839 T589 oasc.CoreContainer.<init> New CoreContainer 303006333
   [junit4]   2> 167840 T589 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/]
   [junit4]   2> 167841 T589 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 167842 T589 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 167842 T589 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 167842 T589 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 167843 T589 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 167843 T589 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 167844 T589 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 167844 T589 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 167844 T589 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 167845 T589 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 167845 T589 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 167846 T589 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 167846 T589 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 167847 T589 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 167847 T589 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 167848 T589 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:64362/solr
   [junit4]   2> 167848 T589 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 167849 T589 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 167850 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 167852 T612 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@195ad25e name:ZooKeeperConnection Watcher:127.0.0.1:64362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 167853 T589 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 167854 T589 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 167856 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 167858 T615 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2816ba97 name:ZooKeeperConnection Watcher:127.0.0.1:64362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 167858 T589 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 167860 T589 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 167863 T589 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 167865 T589 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 167867 T589 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 167870 T589 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 167872 T589 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 167874 T589 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64365_
   [junit4]   2> 167874 T589 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64365_
   [junit4]   2> 167877 T589 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 167879 T589 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 167881 T589 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 167882 T589 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:64365_
   [junit4]   2> 167883 T589 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 167885 T589 oasc.Overseer.start Overseer (id=93188271486074883-127.0.0.1:64365_-n_0000000000) starting
   [junit4]   2> 167887 T589 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 167894 T589 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 167894 T617 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 167895 T589 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 167897 T589 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 167899 T589 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 167901 T616 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 167902 T589 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores
   [junit4]   2> 167905 T589 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores/collection1/
   [junit4]   2> 167905 T589 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 167907 T619 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 167907 T619 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 167908 T619 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 167908 T615 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 167911 T616 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64365",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:64365_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 167912 T616 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64365",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:64365_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 167912 T616 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 167912 T616 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 167914 T615 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 168910 T619 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 168911 T619 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 168912 T619 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 168912 T619 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 168913 T619 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 168914 T619 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores/collection1/'
   [junit4]   2> 168962 T619 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 168966 T619 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 168986 T619 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 169012 T619 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 169067 T619 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 169069 T619 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 169086 T619 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 169688 T619 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 169707 T619 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 169711 T619 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 169752 T619 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169778 T619 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169783 T619 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169791 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169792 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169792 T619 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169793 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169793 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169811 T619 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 169811 T619 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 169812 T619 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/tempDir-001/control/data/
   [junit4]   2> 169812 T619 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2731e5f
   [junit4]   2> 169813 T619 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/tempDir-001/control/data
   [junit4]   2> 169814 T619 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/tempDir-001/control/data/index/
   [junit4]   2> 169814 T619 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 169815 T619 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FB2D131DA8C19148-001/tempDir-001/control/data/index
   [junit4]   2> 169815 T619 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.4736328125, noCFSRatio=1.0]
   [junit4]   2> 169816 T619 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56c63c21 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7095c708),segFN=segments_1,generation=1}
   [junit4]   2> 169817 T619 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169822 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 169822 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 169823 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 169823 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169824 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169824 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 169824 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169825 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169825 T619 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 169826 T619 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 169826 T619 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 169827 T619 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 169828 T619 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 169828 T619 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 169829 T619 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 169829 T619 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 169830 T619 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 169830 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 169832 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 169833 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 169833 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 169834 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 169834 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 169835 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 169836 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 169836 T619 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 169837 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 169859 T619 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169861 T619 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169863 T619 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169866 T619 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169868 T619 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 169872 T619 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 169872 T619 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 169873 T619 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=35, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24559195337800538]
   [junit4]   2> 169874 T619 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56c63c21 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7095c708),segFN=segments_1,generation=1}
   [junit4]   2> 169874 T619 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169875 T619 oass.SolrIndexSearcher.<init> Opening Searcher@7a4c1134[collection1] main
   [junit4]   2> 169875 T619 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 169876 T619 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 169877 T619 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 169877 T619 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 169877 T619 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 169878 T619 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169878 T619 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 169879 T619 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169879 T619 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 169879 T619 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 169880 T620 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a4c1134[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 169881 T619 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 169882 T623 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64365 collection:control_collection shard:shard1
   [junit4]   2> 169883 T589 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 169883 T589 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 169883 T623 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 169888 T589 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 169889 T623 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 169889 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 169891 T627 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7342e709 name:ZooKeeperConnection Watcher:127.0.0.1:64362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 169891 T615 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169892 T623 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 169892 T589 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 169892 T623 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 169893 T616 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 169893 T589 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C278 name=collection1 org.apache.solr.core.SolrCore@27404c18 url=http://127.0.0.1:64365/collection1 node=127.0.0.1:64365_ C278_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:64365, node_name=127.0.0.1:64365_, core=collection1}
   [junit4]   2> 169893 T623 C278 P64365 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64365/collection1/
   [junit4]   2> 169894 T623 C278 P64365 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 169895 T623 C278 P64365 oasc.SyncStrategy.syncToMe http://127.0.0.1:64365/collection1/ has no replicas
   [junit4]   2> 169895 T589 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 169895 T615 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 169895 T623 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64365/collection1/ shard1
   [junit4]   2> 169896 T623 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 169899 T589 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 169899 T589 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 169899 T589 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 169900 T589 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 169901 T630 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged f

[...truncated too long message...]

78 T9936 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:58208/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=testDistribSearch -Dtests.seed=FB2D131DA8C19148 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_NZ -Dtests.timezone=CAT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   72.7s J1 | ReplicationFactorTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FB2D131DA8C19148:7ACB9D05DF9EF174]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:864)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:729)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:545)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:542)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1992687 T9936 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest FB2D131DA8C19148-001
   [junit4]   2> 72677 T9935 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=FST50, multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), text=PostingsFormat(name=Memory doPackFST= false), range_facet_l=Lucene50(blocksize=128), intDefault=FST50, a_t=Lucene50(blocksize=128), id=BlockTreeOrds(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=en_NZ, timezone=CAT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=286561376,total=513802240
   [junit4]   2> NOTE: All tests run in this JVM: [TestSortingResponseWriter, FullSolrCloudDistribCmdsTest, TestPhraseSuggestions, TestClusterStateMutator, TestSolr4Spatial, CopyFieldTest, RecoveryAfterSoftCommitTest, TestSolr4Spatial2, ConnectionManagerTest, DistanceUnitsTest, SolrInfoMBeanTest, WordBreakSolrSpellCheckerTest, TestDistributedMissingSort, SuggestComponentTest, AsyncMigrateRouteKeyTest, HighlighterTest, SortByFunctionTest, HdfsSyncSliceTest, CollectionsAPIAsyncDistributedZkTest, LoggingHandlerTest, TestZkChroot, CursorPagingTest, ConvertedLegacyTest, TestOmitPositions, TestShardHandlerFactory, ResourceLoaderTest, TestDefaultSearchFieldResource, TestSolrDeletionPolicy1, CloudExitableDirectoryReaderTest, TestStressLucene, TestDynamicFieldResource, BJQParserTest, SchemaVersionSpecificBehaviorTest, UniqFieldsUpdateProcessorFactoryTest, LeaderFailoverAfterPartitionTest, TestExpandComponent, TestQueryUtils, TestConfig, PreAnalyzedFieldTest, FieldAnalysisRequestHandlerTest, JSONWriterTest, TestConfigReload, TestFastOutputStream, SolrXmlInZkTest, TestBlobHandler, TestRecovery, TestLeaderElectionZkExpiry, TestLFUCache, CircularListTest, TestExceedMaxTermLength, ZkControllerTest, TestRandomMergePolicy, AutoCommitTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, ShardRoutingTest, ZkCLITest, TestFaceting, TestStressReorder, TestStressVersions, DistributedTermsComponentTest, TestCoreContainer, SimpleFacetsTest, StatsComponentTest, QueryElevationComponentTest, TestSort, TestFunctionQuery, BasicFunctionalityTest, DirectUpdateHandlerTest, OverseerCollectionProcessorTest, SimplePostToolTest, TestCoreDiscovery, SpellCheckCollatorTest, SolrRequestParserTest, TestTrie, SuggesterWFSTTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, QueryEqualityTest, TestRemoteStreaming, CacheHeaderTest, TestSurroundQueryParser, TestWriterPerf, DocumentAnalysisRequestHandlerTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, RequiredFieldsTest, TestSolrQueryParser, FastVectorHighlighterTest, IndexSchemaTest, ReturnFieldsTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, SearchHandlerTest, SOLR749Test, TestSolrIndexConfig, ResponseLogComponentTest, TestSolrDeletionPolicy2, MultiTermTest, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, DirectSolrConnectionTest, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, OpenExchangeRatesOrgProviderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 73.22s, 1 test, 1 error <<< FAILURES!

[...truncated 342 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1846 tests, 6 errors, 55 ignored (8 assumptions)

Total time: 147 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 730643020 bytes
Compression is 0.0%
Took 2 min 13 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 740 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/740/

4 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A40ECF0BF65E3F19:25E8411381015F25]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A40ECF0BF65E3F19:25E8411381015F25]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:44508/xr_gzq/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44508/xr_gzq/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([A40ECF0BF65E3F19:25E8411381015F25]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A40ECF0BF65E3F19:25E8411381015F25]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9682 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/init-core-data-001
   [junit4]   2> 50297 T44 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 50303 T44 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 50342 T44 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 50354 T45 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 50767 T44 oasc.ZkTestServer.run start zk server on port:37149
   [junit4]   2> 50991 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 51336 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51498 T52 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@165b1239 name:ZooKeeperConnection Watcher:127.0.0.1:37149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51499 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51500 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 51508 T44 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 51608 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 51609 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51611 T55 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@240e8a19 name:ZooKeeperConnection Watcher:127.0.0.1:37149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51612 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51612 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 51625 T44 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 51629 T44 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 51633 T44 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 51636 T44 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 51641 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 51642 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 51648 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 51648 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 51653 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 51653 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 51658 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 51658 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 51662 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 51662 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 51665 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 51666 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 51669 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 51670 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 51673 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 51674 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 51677 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 51678 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 51681 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 51682 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 51685 T44 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 51685 T44 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 53970 T44 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores/collection1
   [junit4]   2> 54895 T44 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 55316 T44 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37162
   [junit4]   2> 55514 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 55515 T44 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 55515 T44 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001
   [junit4]   2> 55516 T44 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/'
   [junit4]   2> 55564 T44 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/solr.xml
   [junit4]   2> 55623 T44 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores
   [junit4]   2> 55624 T44 oasc.CoreContainer.<init> New CoreContainer 202401851
   [junit4]   2> 55625 T44 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/]
   [junit4]   2> 55721 T44 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 55722 T44 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 55722 T44 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 55722 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 55723 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 55723 T44 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 55724 T44 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 55724 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 55725 T44 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 55725 T44 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 55726 T44 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 55727 T44 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 55727 T44 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55728 T44 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55728 T44 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 55729 T44 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37149/solr
   [junit4]   2> 55790 T44 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 55790 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55792 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55795 T68 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ff0c55a name:ZooKeeperConnection Watcher:127.0.0.1:37149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55796 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55797 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 55825 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55827 T71 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4df3f011 name:ZooKeeperConnection Watcher:127.0.0.1:37149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55828 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55851 T44 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 55869 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 55880 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 55885 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 55890 T44 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 55917 T44 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 55920 T44 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37161_
   [junit4]   2> 55930 T44 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37161_
   [junit4]   2> 55935 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 55939 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 55942 T44 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 55970 T44 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37161_
   [junit4]   2> 55971 T44 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 55974 T44 oasc.Overseer.start Overseer (id=93182592857538563-127.0.0.1:37161_-n_0000000000) starting
   [junit4]   2> 56001 T44 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 56060 T44 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 56094 T73 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 56095 T44 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 56099 T44 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 56102 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 56142 T72 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 56144 T44 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores
   [junit4]   2> 56181 T44 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores/collection1/
   [junit4]   2> 56182 T44 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 56184 T75 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 56185 T75 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 56206 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 56206 T75 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 56222 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37161_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37161",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 56308 T72 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37161_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37161",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 56319 T72 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 56386 T72 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 56410 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 57208 T75 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 57208 T75 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 57210 T75 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 57210 T75 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 57212 T75 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 57212 T75 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores/collection1/'
   [junit4]   2> 57307 T75 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 57346 T75 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 57401 T75 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 57440 T75 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 57513 T75 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57515 T75 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 57534 T75 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 58031 T75 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 58049 T75 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 58053 T75 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 58090 T75 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58097 T75 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58103 T75 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 58105 T75 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 58105 T75 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 58105 T75 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 58107 T75 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 58107 T75 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 58108 T75 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 58108 T75 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 58109 T75 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/control/data/
   [junit4]   2> 58110 T75 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77682363
   [junit4]   2> 58135 T75 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/control/data
   [junit4]   2> 58136 T75 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/control/data/index/
   [junit4]   2> 58136 T75 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 58137 T75 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/control/data/index
   [junit4]   2> 58137 T75 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=59.951171875, floorSegmentMB=0.953125, forceMergeDeletesPctAllowed=17.099976648494902, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 58139 T75 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@724bf6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d93d26b; maxCacheMB=0.484375 maxMergeSizeMB=0.9169921875)),segFN=segments_1,generation=1}
   [junit4]   2> 58139 T75 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58150 T75 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 58274 T75 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 58275 T75 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 58275 T75 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58275 T75 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58276 T75 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 58276 T75 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58277 T75 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58278 T75 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 58291 T75 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58293 T75 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58294 T75 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58295 T75 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58297 T75 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 58298 T75 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 58299 T75 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 58300 T75 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 58302 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 58304 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 58305 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 58306 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 58307 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 58309 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 58310 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 58311 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 58312 T75 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 58314 T75 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 58348 T75 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58351 T75 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58353 T75 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58356 T75 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58359 T75 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 58371 T75 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 58372 T75 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 58373 T75 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 58374 T75 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@724bf6e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d93d26b; maxCacheMB=0.484375 maxMergeSizeMB=0.9169921875)),segFN=segments_1,generation=1}
   [junit4]   2> 58375 T75 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58375 T75 oass.SolrIndexSearcher.<init> Opening Searcher@6c850b2a[collection1] main
   [junit4]   2> 58376 T75 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 58378 T75 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 58378 T75 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 58398 T75 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 58398 T75 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 58398 T75 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58399 T75 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 58400 T75 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58400 T75 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 58401 T75 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 58404 T76 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6c850b2a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 58410 T75 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 58412 T79 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37161 collection:control_collection shard:shard1
   [junit4]   2> 58412 T44 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 58413 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 58471 T79 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 58483 T79 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 58486 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58487 T79 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 58487 T79 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 58488 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C17 name=collection1 org.apache.solr.core.SolrCore@3ce928bf url=http://127.0.0.1:37161/collection1 node=127.0.0.1:37161_ C17_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:37161_, base_url=http://127.0.0.1:37161, core=collection1, state=down}
   [junit4]   2> 58502 T79 C17 P37161 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37161/collection1/
   [junit4]   2> 58503 T79 C17 P37161 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 58503 T79 C17 P37161 oasc.SyncStrategy.syncToMe http://127.0.0.1:37161/collection1/ has no replicas
   [junit4]   2> 58504 T79 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37161/collection1/ shard1
   [junit4]   2> 58507 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58526 T79 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 58536 T81 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58538 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37161",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 58571 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 58573 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 58575 T84 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@452ed8bd name:ZooKeeperConnection Watcher:127.0.0.1:37149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 58576 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 58577 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 58579 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 58588 T44 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 58645 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58691 T79 oasc.ZkController.register We are http://127.0.0.1:37161/collection1/ and leader is http://127.0.0.1:37161/collection1/
   [junit4]   2> 58691 T79 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37161
   [junit4]   2> 58691 T79 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 58692 T79 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 58692 T79 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 58695 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58700 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37161_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37161",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 58702 T72 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37161_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37161",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 58809 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58809 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 60556 T44 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores/collection1
   [junit4]   2> 60560 T44 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 60562 T44 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 60565 T44 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37179
   [junit4]   2> 60566 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 60566 T44 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 60567 T44 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001
   [junit4]   2> 60567 T44 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/'
   [junit4]   2> 60629 T44 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/solr.xml
   [junit4]   2> 60702 T44 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores
   [junit4]   2> 60703 T44 oasc.CoreContainer.<init> New CoreContainer 350661411
   [junit4]   2> 60704 T44 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/]
   [junit4]   2> 60705 T44 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 60706 T44 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 60706 T44 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 60707 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 60707 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 60707 T44 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 60708 T44 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 60708 T44 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 60709 T44 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 60709 T44 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 60710 T44 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 60711 T44 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 60711 T44 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 60712 T44 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 60712 T44 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 60713 T44 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37149/solr
   [junit4]   2> 60713 T44 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 60714 T44 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 60715 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60717 T96 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e9862db name:ZooKeeperConnection Watcher:127.0.0.1:37149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60718 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60719 T44 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 60722 T44 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60724 T99 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d2afb98 name:ZooKeeperConnection Watcher:127.0.0.1:37149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60724 T44 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60735 T44 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 61743 T44 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37178_
   [junit4]   2> 61744 T44 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37178_
   [junit4]   2> 61751 T44 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 61763 T44 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores
   [junit4]   2> 61766 T44 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores/collection1/
   [junit4]   2> 61766 T44 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 61768 T100 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 61768 T100 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 61770 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 61770 T100 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 61772 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37178_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37178",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 61773 T72 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37178_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37178",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 61773 T72 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 61773 T72 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 61776 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 61777 T99 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 61776 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 62771 T100 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 62772 T100 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 62773 T100 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 62774 T100 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 62775 T100 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 62776 T100 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores/collection1/'
   [junit4]   2> 62798 T100 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 62803 T100 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 62820 T100 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 62848 T100 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 62905 T100 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 62907 T100 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 62928 T100 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 63409 T100 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 63427 T100 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 63431 T100 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 63465 T100 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63470 T100 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63475 T100 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 63477 T100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 63477 T100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 63478 T100 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 63479 T100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 63479 T100 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 63479 T100 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 63479 T100 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 63480 T100 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/jetty1/
   [junit4]   2> 63480 T100 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77682363
   [junit4]   2> 63481 T100 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/jetty1
   [junit4]   2> 63481 T100 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/jetty1/index/
   [junit4]   2> 63481 T100 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 63482 T100 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderFailoverAfterPartitionTest A40ECF0BF65E3F19-001/tempDir-001/jetty1/index
   [junit4]   2> 63482 T100 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=59.951171875, floorSegmentMB=0.953125, forceMergeDeletesPctAllowed=17.099976648494902, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 63483 T100 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@28aa7c3b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50c21a58; maxCacheMB=0.484375 maxMergeSizeMB=0.9169921875)),segFN=segments_1,generation=1}
   [junit4]   2> 63483 T100 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 63492 T100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 63493 T100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 63493 T100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 63493 T100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63493 T100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63494 T100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 63494 T100 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63494 T100 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63495 T100 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 63496 T100 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63497 T100 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63499 T100 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63500 T100 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63501 T100 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 63502 T100 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 63518 T100 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 63520 T100 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 63521 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 63523 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 63524 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 63525 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 63527 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 63528 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 63529 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 63531 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 63532 T100 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 63533 T100 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 63556 T100 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 63558 T100 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 63560 T100 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 63562 T100 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 63564 T100 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 63570 T100 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 63570 T100 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 63572 T100 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 63573 T100 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@28aa7c3b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50c21a58; maxCacheMB=0.484375 maxMergeSizeMB=0.9169921875)),segFN=segments_1,generation=1}
   [junit4]   2> 63573 T100 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 63574 T100 oass.SolrIndexSearcher.<init> Opening Searcher@a213859[collection1] main
   [junit4]   2> 63574 T100 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 63575 T100 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 63575 T100 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 63576 T100 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 63576 T100 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 63576 T100 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 63577 T100 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 63578 T100 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 63578 T100 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 63578 T100 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 63581 T101 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a213859[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 63582 T100 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 63583 T104 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37178 collection:collection1 shard:shard1
   [junit4]   2> 63584 T44 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 63584 T44 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 63584 T104 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 63592 T104 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 63595 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 63596 T104 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 63596 T104 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C18 name=collection1 org.apache.solr.core.SolrCore@6b55473b url=http://127.0.0.1:37178/collection1 node=127.0.0.1:37178_ C18_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:37178_, base_url=http://127.0.0.1:37178, core=collection1, state=down}
   [junit4]   2> 63596 T104 C18 P37178 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37178/collection1/
   [junit4]   2> 63597 T104 C18 P37178 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 63597 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 63597 T104 C18 P37178 oasc.SyncStrategy.syncToMe http://127.0.0.1:37178/collection1/ has no replicas
   [junit4]   2> 63598 T104 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37178/collection1/ shard1
   [junit4]   2> 63598 T104 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 63600 T99 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63600 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63600 T71 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63607 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37178",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 63611 T81 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 63613 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63613 T99 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63658 T104 oasc.ZkController.register We are http://127.0.0.1:37178/collection1/ and leader is http://127.0.0.1:37178/collection1/
   [junit4]   2> 63659 T104 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37178
   [junit4]   2> 63659 T104 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 63659 T104 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 63659 T104 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 63662 T71 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 63663 T72 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37178_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37178",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 7
   [junit4]   2> 63665 T72 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37178_",


[...truncated too long message...]

cketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33855/
   [junit4]   2> 2247694 T10989 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33895/
   [junit4]   2> 2247694 T10989 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33935/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=testDistribSearch -Dtests.seed=A40ECF0BF65E3F19 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pl_PL -Dtests.timezone=Europe/Monaco -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.6s J1 | ReplicationFactorTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A40ECF0BF65E3F19:25E8411381015F25]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:878)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2247703 T10989 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest A40ECF0BF65E3F19-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {text=Lucene50(blocksize=128), id=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=pl_PL, timezone=Europe/Monaco
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=202441720,total=500170752
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedSuggestComponentTest, TestLMJelinekMercerSimilarityFactory, TestCloudManagedSchemaConcurrent, TestFieldTypeCollectionResource, TestFieldSortValues, PingRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, CopyFieldTest, TestPhraseSuggestions, SimpleCollectionCreateDeleteTest, TestFileDictionaryLookup, TestSweetSpotSimilarityFactory, TestSystemIdResolver, SpatialRPTFieldTypeTest, ZkControllerTest, TestMaxScoreQueryParser, DeleteInactiveReplicaTest, PluginInfoTest, DeleteShardTest, SuggestComponentTest, TestConfig, DistributedQueryComponentCustomSortTest, TimeZoneUtilsTest, HdfsDirectoryTest, TestLFUCache, WordBreakSolrSpellCheckerTest, TestAnalyzeInfixSuggestions, AnalyticsQueryTest, CloudMLTQParserTest, EchoParamsTest, ChangedSchemaMergeTest, TestNRTOpen, OverseerRolesTest, CustomCollectionTest, TestReplicationHandlerBackup, FieldMutatingUpdateProcessorTest, DistributedIntervalFacetingTest, TestConfigSets, AutoCommitTest, CoreAdminRequestStatusTest, SolrTestCaseJ4Test, TestSuggestSpellingConverter, OpenCloseCoreStressTest, DeleteReplicaTest, BasicZkTest, TestStressLucene, DistanceUnitsTest, CoreMergeIndexesAdminHandlerTest, IndexSchemaTest, TermVectorComponentTest, QueryElevationComponentTest, TestManagedSchemaFieldTypeResource, TestDefaultSearchFieldResource, TestSolrJ, TestFaceting, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestJoin, TestStressVersions, TestRangeQuery, TestSolr4Spatial, SpellCheckComponentTest, SolrCmdDistributorTest, ConvertedLegacyTest, TestFiltering, TestBadConfig, OverseerCollectionProcessorTest, HighlighterTest, SoftAutoCommitTest, AnalysisAfterCoreReloadTest, SuggesterFSTTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, SpatialFilterTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, DocValuesMultiTest, SolrInfoMBeanTest, TestQueryUtils, DirectSolrSpellCheckerTest, FileBasedSpellCheckerTest, TestValueSourceCache, RequiredFieldsTest, TestSolrQueryParser, LoggingHandlerTest, MBeansHandlerTest, TestBinaryResponseWriter, AlternateDirectoryTest, ResponseLogComponentTest, TestSearchPerf, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, DirectSolrConnectionTest, TestPerFieldSimilarity, TestJmxMonitoredMap, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest, DistributedQueueTest, ExternalCollectionsTest, HttpPartitionTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.66s, 1 test, 1 error <<< FAILURES!

[...truncated 258 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1858 tests, 4 errors, 56 ignored (8 assumptions)

Total time: 138 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 678117619 bytes
Compression is 0.0%
Took 2 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 739 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/739/

6 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest:     1) Thread[id=600, name=OverseerThreadFactory-155-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1840)         at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1722)         at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:621)         at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2859)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest: 
   1) Thread[id=600, name=OverseerThreadFactory-155-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1840)
        at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1722)
        at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:621)
        at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2859)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE]:0)


REGRESSION:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
Could not successfully add blob after 150 attempts. Expecting 2 items. time elapsed 15,833  output  for url is {   "responseHeader":{     "status":0,     "QTime":0},   "response":{     "numFound":1,     "start":0,     "docs":[{         "id":"test/1",         "md5":"83359547e8172e3423e406407145dd52",         "blobName":"test",         "version":1,         "timestamp":"2015-01-20T16:19:04.698Z",         "size":5394}]}}

Stack Trace:
java.lang.AssertionError: Could not successfully add blob after 150 attempts. Expecting 2 items. time elapsed 15,833  output  for url is {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "response":{
    "numFound":1,
    "start":0,
    "docs":[{
        "id":"test/1",
        "md5":"83359547e8172e3423e406407145dd52",
        "blobName":"test",
        "version":1,
        "timestamp":"2015-01-20T16:19:04.698Z",
        "size":5394}]}}
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE:49C73E4E79BE1592]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestBlobHandler.postAndCheck(TestBlobHandler.java:150)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:115)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:71)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37858/saqw/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37858/saqw/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE:49C73E4E79BE1592]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37858/saqw/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38104/qw/c8n_1x3_lf_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38104/qw/c8n_1x3_lf_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE:49C73E4E79BE1592]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor60.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38104/qw/c8n_1x3_lf_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 46 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:38210/nebj/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38210/nebj/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE:49C73E4E79BE1592]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11605/sn_/zx/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11605/sn_/zx/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C821B0560EE175AE:49C73E4E79BE1592]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11605/sn_/zx/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9756 lines...]
   [junit4] Suite: org.apache.solr.cloud.MultiThreadedOCPTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/init-core-data-001
   [junit4]   2> 120914 T416 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 120914 T416 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /tpcac/
   [junit4]   2> 120921 T416 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 120922 T416 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 120923 T417 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 121023 T416 oasc.ZkTestServer.run start zk server on port:39841
   [junit4]   2> 121024 T416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 121025 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121029 T424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4af09a2f name:ZooKeeperConnection Watcher:127.0.0.1:39841 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121029 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121030 T416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 121030 T416 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 121032 T416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 121034 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121035 T427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@662ff271 name:ZooKeeperConnection Watcher:127.0.0.1:39841/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121035 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121036 T416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 121036 T416 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 121039 T416 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 121041 T416 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 121043 T416 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 121046 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 121046 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 121050 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 121051 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 121054 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 121055 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 121057 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 121058 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 121060 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 121061 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 121064 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 121064 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 121067 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 121068 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 121070 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 121071 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 121074 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 121075 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 121077 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 121078 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 121080 T416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 121081 T416 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 122745 T416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 122749 T416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39846
   [junit4]   2> 122749 T416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 122750 T416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 122750 T416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002
   [junit4]   2> 122750 T416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/'
   [junit4]   2> 122781 T416 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/solr.xml
   [junit4]   2> 122834 T416 oasc.CoreContainer.<init> New CoreContainer 946222433
   [junit4]   2> 122835 T416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/]
   [junit4]   2> 122836 T416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 122836 T416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 122836 T416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 122836 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 122837 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 122837 T416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 122837 T416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 122837 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 122837 T416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 122838 T416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 122838 T416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 122838 T416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 122839 T416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 122839 T416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 122839 T416 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 122840 T416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39841/solr
   [junit4]   2> 122840 T416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 122840 T416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 122841 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 122842 T439 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69585ed4 name:ZooKeeperConnection Watcher:127.0.0.1:39841 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 122843 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 122843 T416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 122845 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 122846 T442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e376da5 name:ZooKeeperConnection Watcher:127.0.0.1:39841/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 122846 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 122848 T416 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 122850 T416 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 122853 T416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 122855 T416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 122857 T416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 122860 T416 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 122861 T416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39846_tpcac
   [junit4]   2> 122862 T416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39846_tpcac
   [junit4]   2> 122864 T416 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 122865 T416 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 122867 T416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 122868 T416 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:39846_tpcac
   [junit4]   2> 122869 T416 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 122870 T416 oasc.Overseer.start Overseer (id=93177030756139011-127.0.0.1:39846_tpcac-n_0000000000) starting
   [junit4]   2> 122872 T416 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 122879 T416 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 122880 T444 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 122880 T416 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 122882 T416 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 122883 T416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 122886 T443 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 122890 T446 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 122890 T446 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 122891 T442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 122891 T446 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 122894 T443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:39846_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39846/tpcac",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 122894 T443 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:39846_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39846/tpcac",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 122895 T443 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 122899 T443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 122900 T442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123892 T446 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 123893 T446 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 123894 T446 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 123894 T446 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 123895 T446 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 123895 T446 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/'
   [junit4]   2> 123897 T446 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 123898 T446 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 123899 T446 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 123935 T446 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 123938 T446 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 123952 T446 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 123974 T446 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 124020 T446 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 124022 T446 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 124035 T446 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 124403 T446 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 124428 T446 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 124431 T446 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 124450 T446 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 124455 T446 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 124459 T446 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 124460 T446 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 124461 T446 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 124461 T446 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 124462 T446 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 124462 T446 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 124462 T446 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 124463 T446 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 124463 T446 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/data/
   [junit4]   2> 124463 T446 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@342fe5cf
   [junit4]   2> 124464 T446 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/data
   [junit4]   2> 124464 T446 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/data/index/
   [junit4]   2> 124465 T446 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 124466 T446 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-002/collection1/data/index
   [junit4]   2> 124467 T446 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=611151738, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 124480 T446 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62aa4b4b),segFN=segments_1,generation=1}
   [junit4]   2> 124480 T446 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 124487 T446 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 124487 T446 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 124487 T446 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 124488 T446 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 124488 T446 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 124488 T446 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 124489 T446 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 124489 T446 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 124489 T446 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 124490 T446 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 124491 T446 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 124491 T446 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 124492 T446 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 124492 T446 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 124493 T446 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 124493 T446 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 124494 T446 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 124494 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 124496 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 124496 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 124497 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 124498 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 124498 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 124499 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 124499 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 124500 T446 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 124500 T446 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 124519 T446 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 124521 T446 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 124523 T446 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 124525 T446 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 124527 T446 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 124532 T446 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 124532 T446 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 124533 T446 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=15.291015625, floorSegmentMB=2.1435546875, forceMergeDeletesPctAllowed=2.3693355331565877, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 124588 T446 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62aa4b4b),segFN=segments_1,generation=1}
   [junit4]   2> 124588 T446 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 124589 T446 oass.SolrIndexSearcher.<init> Opening Searcher@4c0df3a7[collection1] main
   [junit4]   2> 124589 T446 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 124590 T446 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 124591 T446 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 124591 T446 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 124591 T446 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 124592 T446 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 124592 T446 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 124593 T446 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 124593 T446 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 124593 T446 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 124594 T447 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4c0df3a7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 124595 T446 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 124596 T450 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39846/tpcac collection:control_collection shard:shard1
   [junit4]   2> 124596 T416 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 124597 T416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 124597 T450 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 124598 T416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 124598 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 124600 T453 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5697dde3 name:ZooKeeperConnection Watcher:127.0.0.1:39841/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 124601 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 124601 T416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 124602 T416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 124604 T450 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 124605 T442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 124606 T450 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 124606 T416 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 124606 T450 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 124606 T443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C37 name=collection1 org.apache.solr.core.SolrCore@588c93f url=http://127.0.0.1:39846/tpcac/collection1 node=127.0.0.1:39846_tpcac C37_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:39846_tpcac, core=collection1, base_url=http://127.0.0.1:39846/tpcac, state=down}
   [junit4]   2> 124607 T450 C37 P39846 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39846/tpcac/collection1/
   [junit4]   2> 124607 T450 C37 P39846 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 124607 T450 C37 P39846 oasc.SyncStrategy.syncToMe http://127.0.0.1:39846/tpcac/collection1/ has no replicas
   [junit4]   2> 124608 T450 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39846/tpcac/collection1/ shard1
   [junit4]   2> 124608 T450 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 124608 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124608 T442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124613 T454 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 124614 T443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39846/tpcac",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 124718 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124766 T450 oasc.ZkController.register We are http://127.0.0.1:39846/tpcac/collection1/ and leader is http://127.0.0.1:39846/tpcac/collection1/
   [junit4]   2> 124767 T450 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39846/tpcac
   [junit4]   2> 124767 T450 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 124767 T450 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 124767 T450 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 124769 T442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 124771 T443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:39846_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39846/tpcac",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"2"} current state version: 3
   [junit4]   2> 124772 T443 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:39846_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39846/tpcac",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 124875 T442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124876 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 126251 T416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 126252 T416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 126255 T416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39864
   [junit4]   2> 126256 T416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 126256 T416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 126257 T416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003
   [junit4]   2> 126257 T416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/'
   [junit4]   2> 126308 T416 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/solr.xml
   [junit4]   2> 126397 T416 oasc.CoreContainer.<init> New CoreContainer 358000236
   [junit4]   2> 126397 T416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/]
   [junit4]   2> 126399 T416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 126399 T416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 126399 T416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 126400 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 126400 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 126401 T416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 126401 T416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 126401 T416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 126402 T416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 126402 T416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 126402 T416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 126403 T416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 126404 T416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 126404 T416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 126404 T416 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 126405 T416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39841/solr
   [junit4]   2> 126405 T416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 126405 T416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 126406 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126408 T466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fb9e02a name:ZooKeeperConnection Watcher:127.0.0.1:39841 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126409 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126409 T416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 126412 T416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126413 T469 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fac9d6a name:ZooKeeperConnection Watcher:127.0.0.1:39841/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126413 T416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126419 T416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 127424 T416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39864_tpcac
   [junit4]   2> 127425 T416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39864_tpcac
   [junit4]   2> 127429 T416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 127437 T470 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 127437 T470 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 127438 T442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127438 T470 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 127439 T443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:39864_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39864/tpcac",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"2"} current state version: 4
   [junit4]   2> 127440 T443 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:39864_tpcac",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39864/tpcac",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 127440 T443 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 127441 T443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 127442 T442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127442 T469 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127442 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 128439 T470 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 128440 T470 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 128441 T470 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 128441 T470 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 128442 T470 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 128442 T470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/'
   [junit4]   2> 128444 T470 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 128444 T470 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 128445 T470 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest%20C821B0560EE175AE-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 128480 T470 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 128483 T470 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 128496 T470 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 128517 T470 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 128563 T470 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 128564 T470 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 128577 T470 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 128957 T470 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 128970 T470 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 128973 T470 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 128993 T470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 128998 T470 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 129002 T470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 129003 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 129003 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 129004 T470 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 129005 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 129005 T470 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 129005 T470 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 129005 T470 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 129006 T470 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/data/
   [junit4]   2> 129006 T470 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@342fe5cf
   [junit4]   2> 129007 T470 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/data
   [junit4]   2> 129007 T470 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/data/index/
   [junit4]   2> 129008 T470 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 129010 T470 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/tempDir-003/collection1/data/index
   [junit4]   2> 129011 T470 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=611151738, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 129016 T470 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.MultiThreadedOCPTest C821B0560EE175AE-001/index-MMapDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62aa4b4b),segFN=segments_1,generation=1}
   [junit4]   2> 129016 T470 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 129024 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 129025 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 129025 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 129025 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 129026 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 129026 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 129027 T470 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 129027 T470 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 129027 T470 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 129028 T470 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 129029 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 129030 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 129030 T470 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 129031 T470 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 129032 T470 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 129034 T470 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 129034 T470 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 129035 T470 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 12

[...truncated too long message...]

ected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2231150 T5799 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@357ab475 name:ZooKeeperConnection Watcher:127.0.0.1:18589/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2231150 T5614 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2231151 T5614 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2231151 T5614 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2231153 T5614 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2231154 T5614 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2231156 T5614 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2231157 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2231158 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2231160 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2231161 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2231263 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2231264 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2231266 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2231266 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2231268 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2231269 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2231270 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2231271 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2231273 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2231273 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2231275 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2231276 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2231277 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2231278 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2231280 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2231280 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2231282 T5614 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2231283 T5614 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2231285 T5614 oas.SolrTestCaseJ4.tearDown ###Ending testDown
   [junit4]   2> 2231286 T5614 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18589 18589
   [junit4]   2> 2231481 T5789 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18589 18589
   [junit4]   2> 2231483 T5614 oas.SolrTestCaseJ4.tearDown ###Ending testDown
   [junit4]   2> 2231483 T5614 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18589 18589
   [junit4]   2> 2231484 T5614 oas.SolrTestCaseJ4.tearDown ###Ending testDown
   [junit4]   2> 2231485 T5614 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18589 18589
   [junit4]   2> 2231487 T5614 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading C821B0560EE175AE-001
   [junit4]   2> 52314 T5613 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4274, maxDocsPerChunk=7, blockSize=10), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4274, blockSize=10)), sim=DefaultSimilarity, locale=en_GB, timezone=Europe/Mariehamn
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=284504288,total=381157376
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUCache, UniqFieldsUpdateProcessorFactoryTest, SortByFunctionTest, BlockCacheTest, InfoHandlerTest, DeleteLastCustomShardedReplicaTest, TestBulkSchemaAPI, JSONWriterTest, OutputWriterTest, CoreAdminRequestStatusTest, PathHierarchyTokenizerFactoryTest, TestDocBasedVersionConstraints, ConvertedLegacyTest, ExitableDirectoryReaderTest, OverriddenZkACLAndCredentialsProvidersTest, SuggesterTSTTest, MultiThreadedOCPTest, BadComponentTest, LegacyHTMLStripCharFilterTest, QueryElevationComponentTest, TestStressUserVersions, TestOrdValues, ResourceLoaderTest, TestLRUStatsCache, TestExactSharedStatsCache, DocValuesMissingTest, HdfsBasicDistributedZk2Test, CircularListTest, ExternalFileFieldSortTest, TestTolerantSearch, FieldAnalysisRequestHandlerTest, TestInitQParser, TestSolrDeletionPolicy1, TestHighFrequencyDictionaryFactory, DeleteReplicaTest, PingRequestHandlerTest, DOMUtilTest, DateFieldTest, TestCloudManagedSchema, TestBulkSchemaConcurrent, TestDocSet, TestSolrConfigHandlerCloud, TestSolr4Spatial, DocumentAnalysisRequestHandlerTest, SharedFSAutoReplicaFailoverTest, TestSolrConfigHandlerConcurrent, IndexSchemaTest, TestRecovery, AutoCommitTest, TestSerializedLuceneMatchVersion, MinimalSchemaTest, TestCloudPivotFacet, AnalyticsQueryTest, TestPerFieldSimilarity, DistributedQueryComponentCustomSortTest, TestCharFilters, SimplePostToolTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, ShardRoutingTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, TestDistribDocBasedVersion, TestRebalanceLeaders, TestRequestStatusCollectionAPI, HdfsBasicDistributedZkTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, TestClusterStateMutator, ZkStateWriterTest, CachingDirectoryFactoryTest, HdfsDirectoryFactoryTest, QueryResultKeyTest, TestConfigOverlay, TestDynamicLoading]
   [junit4] Completed on J1 in 53.12s, 2 tests, 1 failure <<< FAILURES!

[...truncated 251 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 463 suites, 1873 tests, 1 suite-level error, 4 errors, 1 failure, 57 ignored (9 assumptions)

Total time: 163 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 656465921 bytes
Compression is 0.0%
Took 2 min 2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 738 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/738/

12 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=734, name=SessionTracker, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=734, name=SessionTracker, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=1149, name=Thread-464, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1149, name=Thread-464, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57723/collection2_shard4_replica3
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:379)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57723/collection2_shard4_replica3
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:376)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57723/collection2_shard4_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 5 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=1148, name=Thread-463, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1148, name=Thread-463, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:379)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:376)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 5 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 10 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=814, name=qtp580129851-814, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=814, name=qtp580129851-814, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)
	at java.util.concurrent.ConcurrentLinkedQueue.offer(ConcurrentLinkedQueue.java:327)
	at java.util.concurrent.ConcurrentLinkedQueue.add(ConcurrentLinkedQueue.java:296)
	at org.eclipse.jetty.io.PooledBuffers.returnBuffer(PooledBuffers.java:106)
	at org.eclipse.jetty.http.HttpParser.returnBuffers(HttpParser.java:1116)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:134)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=1147, name=Thread-462, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1147, name=Thread-462, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.FullSolrCloudDistribCmdsTest

Error Message:
Captured an uncaught exception in thread: Thread[id=1012, name=qtp635027840-1012, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1012, name=qtp635027840-1012, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16998/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16998/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22:1AF2E0F25C454A1E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16998/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57194/_f/q/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57194/_f/q/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22:1AF2E0F25C454A1E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57194/_f/q/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34172/_yl/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34172/_yl/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22:1AF2E0F25C454A1E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23187/vqy/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23187/vqy/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22:1AF2E0F25C454A1E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23187/vqy/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9994 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/init-core-data-001
   [junit4]   2> 1259371 T3030 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /vqy/
   [junit4]   2> 1259375 T3030 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1259376 T3030 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1259377 T3031 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1259476 T3030 oasc.ZkTestServer.run start zk server on port:23183
   [junit4]   2> 1259477 T3030 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1259478 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1259482 T3038 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37a10eca name:ZooKeeperConnection Watcher:127.0.0.1:23183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1259482 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1259482 T3030 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1259482 T3030 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1259486 T3030 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1259487 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1259488 T3041 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e36e483 name:ZooKeeperConnection Watcher:127.0.0.1:23183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1259488 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1259489 T3030 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1259489 T3030 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1259491 T3030 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1259492 T3030 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1259493 T3030 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1259495 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1259495 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1259497 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1259498 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1259600 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1259601 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1259602 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1259603 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1259604 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1259605 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1259607 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1259607 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1259609 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1259610 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1259611 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1259612 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1259613 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1259614 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1259616 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1259616 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1259618 T3030 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1259618 T3030 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1260917 T3030 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1260921 T3030 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23188
   [junit4]   2> 1260922 T3030 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1260922 T3030 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1260922 T3030 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002
   [junit4]   2> 1260923 T3030 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002/'
   [junit4]   2> 1260952 T3030 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002/solr.xml
   [junit4]   2> 1261005 T3030 oasc.CoreContainer.<init> New CoreContainer 178202361
   [junit4]   2> 1261006 T3030 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002/]
   [junit4]   2> 1261006 T3030 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1261007 T3030 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1261007 T3030 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1261007 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1261007 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1261008 T3030 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1261008 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1261008 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1261008 T3030 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1261009 T3030 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1261009 T3030 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1261009 T3030 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1261010 T3030 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1261010 T3030 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1261010 T3030 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1261010 T3030 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23183/solr
   [junit4]   2> 1261011 T3030 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1261011 T3030 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1261012 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1261013 T3053 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@541bc8c7 name:ZooKeeperConnection Watcher:127.0.0.1:23183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1261014 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1261015 T3030 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1261017 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1261019 T3056 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c5ad654 name:ZooKeeperConnection Watcher:127.0.0.1:23183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1261019 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1261021 T3030 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1261023 T3030 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1261025 T3030 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1261027 T3030 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1261029 T3030 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1261031 T3030 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1261032 T3030 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23187_vqy
   [junit4]   2> 1261032 T3030 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23187_vqy
   [junit4]   2> 1261034 T3030 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1261035 T3030 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1261036 T3030 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1261038 T3030 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23187_vqy
   [junit4]   2> 1261038 T3030 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1261039 T3030 oasc.Overseer.start Overseer (id=93171329908473859-127.0.0.1:23187_vqy-n_0000000000) starting
   [junit4]   2> 1261041 T3030 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1261046 T3030 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1261047 T3058 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1261047 T3030 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1261049 T3030 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1261050 T3030 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1261052 T3057 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1261056 T3060 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1261056 T3060 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1261057 T3060 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1261057 T3056 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1261059 T3057 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:23187_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23187/vqy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1261059 T3057 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:23187_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23187/vqy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1261060 T3057 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1261060 T3057 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1261061 T3056 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1262057 T3060 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1262058 T3060 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1262058 T3060 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1262059 T3060 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1262059 T3060 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1262059 T3060 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002/collection1/'
   [junit4]   2> 1262061 T3060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1262062 T3060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1262062 T3060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1262096 T3060 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1262098 T3060 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1262110 T3060 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1262132 T3060 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1262179 T3060 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1262280 T3060 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1262291 T3060 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1262670 T3060 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1262683 T3060 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1262686 T3060 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1262697 T3060 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1262701 T3060 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1262705 T3060 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1262706 T3060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1262706 T3060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1262706 T3060 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1262707 T3060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1262707 T3060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1262707 T3060 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1262708 T3060 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1262708 T3060 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/control/data/
   [junit4]   2> 1262708 T3060 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d6365f5
   [junit4]   2> 1262709 T3060 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/control/data
   [junit4]   2> 1262709 T3060 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/control/data/index/
   [junit4]   2> 1262710 T3060 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1262711 T3060 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/control/data/index
   [junit4]   2> 1262711 T3060 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=0.9130859375, floorSegmentMB=1.3271484375, forceMergeDeletesPctAllowed=15.082689341104693, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1262716 T3060 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4bffe94c),segFN=segments_1,generation=1}
   [junit4]   2> 1262716 T3060 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1262721 T3060 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1262722 T3060 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1262722 T3060 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1262722 T3060 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1262723 T3060 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1262723 T3060 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1262723 T3060 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1262724 T3060 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1262724 T3060 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1262725 T3060 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1262725 T3060 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1262726 T3060 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1262726 T3060 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1262727 T3060 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1262727 T3060 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1262728 T3060 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1262728 T3060 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1262729 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1262730 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1262730 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1262731 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1262736 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1262736 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1262737 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1262738 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1262739 T3060 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1262739 T3060 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1262760 T3060 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1262762 T3060 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1262764 T3060 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1262766 T3060 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1262768 T3060 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1262772 T3060 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1262772 T3060 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1262773 T3060 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.777206772509576]
   [junit4]   2> 1262776 T3060 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4bffe94c),segFN=segments_1,generation=1}
   [junit4]   2> 1262777 T3060 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1262777 T3060 oass.SolrIndexSearcher.<init> Opening Searcher@579ef7a5[collection1] main
   [junit4]   2> 1262778 T3060 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1262778 T3060 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1262779 T3060 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1262779 T3060 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1262779 T3060 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1262780 T3060 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1262780 T3060 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1262780 T3060 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1262781 T3060 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1262781 T3060 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1262781 T3060 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1262782 T3061 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@579ef7a5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1262782 T3064 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23187/vqy collection:control_collection shard:shard1
   [junit4]   2> 1262783 T3030 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1262783 T3030 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1262783 T3064 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1262787 T3064 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1262787 T3030 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1262788 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1262788 T3056 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1262789 T3064 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1262790 T3064 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1262790 T3057 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C618 name=collection1 org.apache.solr.core.SolrCore@8fbaee6 url=http://127.0.0.1:23187/vqy/collection1 node=127.0.0.1:23187_vqy C618_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23187_vqy, core=collection1, base_url=http://127.0.0.1:23187/vqy, state=down}
   [junit4]   2> 1262790 T3064 C618 P23187 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23187/vqy/collection1/
   [junit4]   2> 1262790 T3068 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@309f8743 name:ZooKeeperConnection Watcher:127.0.0.1:23183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1262790 T3064 C618 P23187 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1262791 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1262791 T3064 C618 P23187 oasc.SyncStrategy.syncToMe http://127.0.0.1:23187/vqy/collection1/ has no replicas
   [junit4]   2> 1262791 T3056 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1262792 T3030 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1262792 T3064 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23187/vqy/collection1/ shard1
   [junit4]   2> 1262793 T3064 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1262794 T3030 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1262796 T3030 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1262797 T3069 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1262798 T3057 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23187/vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1262900 T3068 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1262949 T3064 oasc.ZkController.register We are http://127.0.0.1:23187/vqy/collection1/ and leader is http://127.0.0.1:23187/vqy/collection1/
   [junit4]   2> 1262950 T3064 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23187/vqy
   [junit4]   2> 1262950 T3064 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1262950 T3064 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1262950 T3064 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1262951 T3056 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1262953 T3057 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:23187_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23187/vqy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1262953 T3057 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:23187_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23187/vqy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1263056 T3056 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1263056 T3068 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1264025 T3030 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1264026 T3030 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1264029 T3030 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23195
   [junit4]   2> 1264029 T3030 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1264029 T3030 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1264030 T3030 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003
   [junit4]   2> 1264030 T3030 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003/'
   [junit4]   2> 1264060 T3030 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003/solr.xml
   [junit4]   2> 1264113 T3030 oasc.CoreContainer.<init> New CoreContainer 355901276
   [junit4]   2> 1264113 T3030 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003/]
   [junit4]   2> 1264114 T3030 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1264114 T3030 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1264114 T3030 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1264114 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1264115 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1264115 T3030 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1264115 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1264115 T3030 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1264116 T3030 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1264116 T3030 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1264116 T3030 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1264117 T3030 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1264117 T3030 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1264117 T3030 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1264117 T3030 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1264118 T3030 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23183/solr
   [junit4]   2> 1264118 T3030 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1264118 T3030 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1264119 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1264120 T3081 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d17e64e name:ZooKeeperConnection Watcher:127.0.0.1:23183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1264121 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1264122 T3030 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1264124 T3030 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1264126 T3084 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@758f8f8d name:ZooKeeperConnection Watcher:127.0.0.1:23183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1264126 T3030 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1264131 T3030 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1265134 T3030 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23194_vqy
   [junit4]   2> 1265135 T3030 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23194_vqy
   [junit4]   2> 1265138 T3030 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1265145 T3085 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1265145 T3085 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1265146 T3085 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1265146 T3056 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1265147 T3057 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:23194_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23194/vqy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 1265148 T3057 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:23194_vqy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23194/vqy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1265148 T3057 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1265148 T3057 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1265149 T3084 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1265149 T3068 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1265149 T3056 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1266147 T3085 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1266148 T3085 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1266148 T3085 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1266148 T3085 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1266149 T3085 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1266149 T3085 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003/collection1/'
   [junit4]   2> 1266151 T3085 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1266152 T3085 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1266152 T3085 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%209B146EEA2B1A2A22-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1266197 T3085 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1266199 T3085 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1266211 T3085 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1266235 T3085 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1266288 T3085 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1266389 T3085 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1266401 T3085 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1266784 T3085 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1266797 T3085 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1266800 T3085 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1266811 T3085 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1266815 T3085 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1266819 T3085 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1266820 T3085 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1266821 T3085 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1266821 T3085 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1266822 T3085 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1266822 T3085 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1266822 T3085 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1266822 T3085 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1266823 T3085 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/jetty1/
   [junit4]   2> 1266823 T3085 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d6365f5
   [junit4]   2> 1266824 T3085 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B146EEA2B1A2A22-001/tempDir-001/jetty1
   [junit4]   2> 1266824 T3085 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorT

[...truncated too long message...]

equestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 10 moreThrowable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1148, name=Thread-463, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)
   [junit4]    > 	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:379)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:376)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:32494/collection2_shard7_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
   [junit4]    > 	... 5 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 10 moreThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=814, name=qtp580129851-814, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9B146EEA2B1A2A22]:0)
   [junit4]    > 	at java.util.concurrent.ConcurrentLinkedQueue.offer(ConcurrentLinkedQueue.java:327)
   [junit4]    > 	at java.util.concurrent.ConcurrentLinkedQueue.add(ConcurrentLinkedQueue.java:296)
   [junit4]    > 	at org.eclipse.jetty.io.PooledBuffers.returnBuffer(PooledBuffers.java:106)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.returnBuffers(HttpParser.java:1116)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:134)
   [junit4]    > 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1147, name=Thread-462, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceededThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1012, name=qtp635027840-1012, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Completed on J1 in 7224.90s, 1 test, 8 errors <<< FAILURES!

[...truncated 79 lines...]
   [junit4] JVM J1: stdout was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/temp/junit4-J1-20150119_151324_935.sysout
   [junit4] >>> JVM J1: stdout (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/heapdumps/java_pid58826.hprof ...
   [junit4] Heap dump file created [578765735 bytes in 8.579 secs]
   [junit4] <<< JVM J1: EOF ----

   [junit4] JVM J1: stderr was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/temp/junit4-J1-20150119_151324_935.syserr
   [junit4] >>> JVM J1: stderr (verbatim) ----
   [junit4] 
   [junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-463"
   [junit4] <<< JVM J1: EOF ----

[...truncated 16 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 463 suites, 1873 tests, 7 suite-level errors, 5 errors, 172 ignored (9 assumptions)

Total time: 216 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 6 artifacts
Archive block size is 32768
Received 0 blocks and 1232947641 bytes
Compression is 0.0%
Took 4 min 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 737 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/737/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:11245/ywx/h: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:534)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:910)  at org.apache.solr.cloud.ZkController.register(ZkController.java:812)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:469)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:522)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:489)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:585)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:740)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /ywx/h/admin/cores. Reason: <pre>    {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:534)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)  at org.apache.solr.cloud.ZkController.register(ZkController.java:910)  at org.apache.solr.cloud.ZkController.register(ZkController.java:812)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:469)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:522)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:489)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:585)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:740)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:11245/ywx/h: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:534)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:910)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:812)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:469)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:522)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:489)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:585)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:740)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /ywx/h/admin/cores. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:534)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:258)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:910)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:812)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:469)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:522)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:489)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:585)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:740)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:527)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:348)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:467)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:203)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16408/pp_qwu/n/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16408/pp_qwu/n/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([33C05D2EECFC3F7E:B226D3369BA35F42]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16408/pp_qwu/n/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37841/juiv/bm/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37841/juiv/bm/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([33C05D2EECFC3F7E:B226D3369BA35F42]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37841/juiv/bm/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37639/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37639/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([33C05D2EECFC3F7E:B226D3369BA35F42]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor65.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 47 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37965/km_/a/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37965/km_/a/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([33C05D2EECFC3F7E:B226D3369BA35F42]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37965/km_/a/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9715 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/init-core-data-001
   [junit4]   2> 169585 T706 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pp_qwu/n
   [junit4]   2> 169592 T706 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 169593 T706 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 169594 T707 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 169694 T706 oasc.ZkTestServer.run start zk server on port:16367
   [junit4]   2> 169695 T706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 169696 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 169701 T714 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1556a66c name:ZooKeeperConnection Watcher:127.0.0.1:16367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 169702 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 169703 T706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 169704 T706 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 169709 T706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 169710 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 169713 T717 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d7c869c name:ZooKeeperConnection Watcher:127.0.0.1:16367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 169713 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 169745 T706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 169746 T706 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 169750 T706 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 169754 T706 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 169758 T706 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 169763 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 169764 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 169771 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 169772 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 169777 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 169777 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 169782 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 169782 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 169787 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 169788 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 169792 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 169794 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 169798 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 169799 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 169803 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 169804 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 169809 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 169810 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 169814 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 169815 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 169820 T706 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 169820 T706 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 171751 T706 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 171756 T706 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16388
   [junit4]   2> 171757 T706 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 171757 T706 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 171758 T706 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002
   [junit4]   2> 171758 T706 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002/'
   [junit4]   2> 171821 T706 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002/solr.xml
   [junit4]   2> 171918 T706 oasc.CoreContainer.<init> New CoreContainer 232102565
   [junit4]   2> 171919 T706 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002/]
   [junit4]   2> 171920 T706 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 171921 T706 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 171921 T706 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 171922 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 171923 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 171923 T706 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 171923 T706 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 171924 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 171924 T706 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 171925 T706 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 171925 T706 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 171926 T706 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 171927 T706 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 171927 T706 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 171928 T706 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 171928 T706 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16367/solr
   [junit4]   2> 171928 T706 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 171929 T706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 171930 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 171932 T729 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f22005f name:ZooKeeperConnection Watcher:127.0.0.1:16367 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 171933 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 171935 T706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 171938 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 171941 T732 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4737bfe name:ZooKeeperConnection Watcher:127.0.0.1:16367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 171941 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 171945 T706 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 171950 T706 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 171956 T706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 171961 T706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 171966 T706 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 171972 T706 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 171974 T706 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16387_pp_qwu%2Fn
   [junit4]   2> 171976 T706 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16387_pp_qwu%2Fn
   [junit4]   2> 171980 T706 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 171984 T706 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 171987 T706 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 171991 T706 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:16387_pp_qwu%2Fn
   [junit4]   2> 171991 T706 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 171995 T706 oasc.Overseer.start Overseer (id=93159950141423619-127.0.0.1:16387_pp_qwu%2Fn-n_0000000000) starting
   [junit4]   2> 171999 T706 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 172011 T706 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 172013 T734 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 172013 T706 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 172017 T706 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 172019 T706 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 172024 T733 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 172031 T736 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 172032 T736 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 172033 T732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 172033 T736 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 172038 T733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16387/pp_qwu/n",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:16387_pp_qwu%2Fn",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 172039 T733 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16387/pp_qwu/n",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:16387_pp_qwu%2Fn",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 172039 T733 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 172039 T733 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 172041 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 173034 T736 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 173035 T736 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 173036 T736 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 173037 T736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 173038 T736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 173039 T736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002/collection1/'
   [junit4]   2> 173042 T736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%2033C05D2EECFC3F7E-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 173043 T736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%2033C05D2EECFC3F7E-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 173044 T736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%2033C05D2EECFC3F7E-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 173085 T736 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 173092 T736 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 173112 T736 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 173142 T736 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 173199 T736 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 173201 T736 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 173215 T736 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 173655 T736 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 173679 T736 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 173682 T736 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 173708 T736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 173715 T736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 173721 T736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 173724 T736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 173725 T736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 173725 T736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 173727 T736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 173728 T736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 173728 T736 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 173728 T736 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 173729 T736 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-001/control/data/
   [junit4]   2> 173729 T736 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7072ba17
   [junit4]   2> 173731 T736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-001/control/data
   [junit4]   2> 173731 T736 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-001/control/data/index/
   [junit4]   2> 173731 T736 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 173732 T736 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-001/control/data/index
   [junit4]   2> 173732 T736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@72b37923
   [junit4]   2> 173733 T736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@64d623a9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26bf5ff7),segFN=segments_1,generation=1}
   [junit4]   2> 173734 T736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 173740 T736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 173740 T736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 173741 T736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 173741 T736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 173741 T736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 173742 T736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 173742 T736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 173743 T736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 173743 T736 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 173744 T736 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173745 T736 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173745 T736 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173746 T736 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173747 T736 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 173747 T736 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 173748 T736 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 173749 T736 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 173749 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 173750 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 173751 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 173752 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 173752 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 173753 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 173754 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 173754 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 173755 T736 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 173755 T736 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 173785 T736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 173789 T736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 173792 T736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 173795 T736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 173798 T736 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 173804 T736 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 173804 T736 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 173805 T736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@30a916b1
   [junit4]   2> 173806 T736 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@64d623a9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26bf5ff7),segFN=segments_1,generation=1}
   [junit4]   2> 173806 T736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 173807 T736 oass.SolrIndexSearcher.<init> Opening Searcher@2444770c[collection1] main
   [junit4]   2> 173807 T736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 173809 T736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 173809 T736 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 173810 T736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 173810 T736 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 173810 T736 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 173811 T736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 173812 T736 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 173812 T736 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 173812 T736 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 173814 T737 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2444770c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 173814 T736 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 173815 T740 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16387/pp_qwu/n collection:control_collection shard:shard1
   [junit4]   2> 173816 T706 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 173817 T706 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 173817 T740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 173821 T706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 173822 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 173825 T744 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@94fcb4b name:ZooKeeperConnection Watcher:127.0.0.1:16367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 173825 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 173826 T706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 173827 T740 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 173828 T706 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 173830 T732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 173831 T740 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 173832 T740 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C64 name=collection1 org.apache.solr.core.SolrCore@7f2bf018 url=http://127.0.0.1:16387/pp_qwu/n/collection1 node=127.0.0.1:16387_pp_qwu%2Fn C64_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:16387/pp_qwu/n, node_name=127.0.0.1:16387_pp_qwu%2Fn, core=collection1, state=down}
   [junit4]   2> 173832 T740 C64 P16387 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16387/pp_qwu/n/collection1/
   [junit4]   2> 173832 T733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 173832 T740 C64 P16387 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 173833 T740 C64 P16387 oasc.SyncStrategy.syncToMe http://127.0.0.1:16387/pp_qwu/n/collection1/ has no replicas
   [junit4]   2> 173833 T740 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16387/pp_qwu/n/collection1/ shard1
   [junit4]   2> 173834 T740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 173834 T706 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 173835 T706 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 173835 T706 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 173835 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 173835 T744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 173836 T706 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 173839 T747 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76a2a989 name:ZooKeeperConnection Watcher:127.0.0.1:16367/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 173839 T706 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 173841 T706 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 173844 T748 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 173846 T733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16387/pp_qwu/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 173851 T733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 2
   [junit4]   2> 173852 T733 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 173853 T744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 173859 T748 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 173861 T733 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 173863 T744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 173896 T740 oasc.ZkController.register We are http://127.0.0.1:16387/pp_qwu/n/collection1/ and leader is http://127.0.0.1:16387/pp_qwu/n/collection1/
   [junit4]   2> 173897 T740 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16387/pp_qwu/n
   [junit4]   2> 173897 T740 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 173897 T740 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 173898 T740 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 173900 T732 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 173904 T733 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16387/pp_qwu/n",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:16387_pp_qwu%2Fn",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 173905 T733 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16387/pp_qwu/n",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:16387_pp_qwu%2Fn",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 174013 T744 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174013 T732 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 175862 T706 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 175863 T706 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 175866 T706 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16395
   [junit4]   2> 175867 T706 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 175868 T706 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 175868 T706 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-003
   [junit4]   2> 175869 T706 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-003/'
   [junit4]   2> 175933 T706 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-003/solr.xml
   [junit4]   2> 176042 T706 oasc.CoreContainer.<init> New CoreContainer 1455551455
   [junit4]   2> 176043 T706 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 33C05D2EECFC3F7E-001/tempDir-003/]
   [junit4]   2> 176044 T706 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 176044 T706 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 176044 T706 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 176045 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 176045 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 176045 T706 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 176045 T706 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 176046 T706 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 176046 T706 oashc.HttpShardHandlerF

[...truncated too long message...]

es
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1883906 T3786 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37949/km_/a
   [junit4]   2> 1883907 T3786 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:37965/km_/a
   [junit4]   2> 1883907 T3786 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:37979/km_/a
   [junit4]   2> 1883907 T3786 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:37938/km_/a
   [junit4]   2> 1883908 T3786 oasc.ReplicationFactorTest.tearDown super.tearDown complete, closing all socket proxies
   [junit4]   2> 1883908 T3786 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37949/km_/a
   [junit4]   2> 1883909 T3786 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37965/km_/a
   [junit4]   2> 1883909 T3786 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37979/km_/a
   [junit4]   2> 1883909 T3786 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37938/km_/a
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=testDistribSearch -Dtests.seed=33C05D2EECFC3F7E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_MT -Dtests.timezone=America/Maceio -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.0s J2 | ReplicationFactorTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37965/km_/a/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([33C05D2EECFC3F7E:B226D3369BA35F42]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37965/km_/a/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
   [junit4]    > 	... 47 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 52 more
   [junit4]   2> 1883946 T3786 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 33C05D2EECFC3F7E-001
   [junit4]   2> 73006 T3785 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), text=FSTOrd50, range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=FST50, _version_=FST50, timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), id=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=en_MT, timezone=America/Maceio
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=245099568,total=362283008
   [junit4]   2> NOTE: All tests run in this JVM: [AnalyticsMergeStrategyTest, TestSolrQueryParser, TestSolrDeletionPolicy2, TestConfig, SolrPluginUtilsTest, DistribCursorPagingTest, TestLazyCores, TestReversedWildcardFilterFactory, ClusterStateTest, TestManagedResource, TestManagedResourceStorage, SimpleMLTQParserTest, SolrRequestParserTest, TestManagedSynonymFilterFactory, TestReload, QueryEqualityTest, TestDynamicFieldCollectionResource, SpatialFilterTest, TestCloudSchemaless, HdfsDirectoryTest, TestCSVResponseWriter, CacheHeaderTest, BasicDistributedZk2Test, DirectUpdateHandlerOptimizeTest, AlternateDirectoryTest, TestClassNameShortening, SpellCheckComponentTest, DateRangeFieldTest, MergeStrategyTest, TestSchemaSimilarityResource, TestDefaultSimilarityFactory, LeaderFailoverAfterPartitionTest, StandardRequestHandlerTest, HdfsLockFactoryTest, StatsComponentTest, TestLMDirichletSimilarityFactory, HighlighterConfigTest, TestAnalyzedSuggestions, MoreLikeThisHandlerTest, HdfsRecoveryZkTest, ResponseLogComponentTest, PrimUtilsTest, TestPHPSerializedResponseWriter, DocumentBuilderTest, TestReRankQParserPlugin, TestMissingGroups, TestSolrQueryParserResource, TestRandomDVFaceting, TestCursorMarkWithoutUniqueKey, TestRemoteStreaming, PeerSyncTest, TestManagedSchemaFieldResource, TestFieldTypeResource, TestPhraseSuggestions, TestShortCircuitedRequests, OverseerTest, TestReplicationHandler, ReplicationFactorTest]
   [junit4] Completed on J2 in 73.29s, 1 test, 1 error <<< FAILURES!

[...truncated 295 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 463 suites, 1873 tests, 5 errors, 57 ignored (9 assumptions)

Total time: 141 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 704111089 bytes
Compression is 0.0%
Took 2 min 23 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 736 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/736/

8 tests failed.
REGRESSION:  org.apache.solr.TestDistributedSearch.testDistribSearch

Error Message:
Captured an uncaught exception in thread: Thread[id=23602, name=Thread-7891, state=RUNNABLE, group=TGRP-TestDistributedSearch]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=23602, name=Thread-7891, state=RUNNABLE, group=TGRP-TestDistributedSearch]
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:49767/t_mu/ab
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078]:0)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:579)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:49767/t_mu/ab
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:302)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:574)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 5 more


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
Error executing query

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Error executing query
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:100)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:302)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:413)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:143)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.ParseException: Invalid content type: 
	at org.apache.http.entity.ContentType.parse(ContentType.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:511)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	... 46 more


REGRESSION:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
Could not successfully add blob after 150 attempts. Expecting 2 items. time elapsed 16.021  output  for url is {   "responseHeader":{     "status":0,     "QTime":0},   "response":{     "numFound":1,     "start":0,     "docs":[{         "id":"test/1",         "md5":"33641e30a627df1efbdb4386e71b05e0",         "blobName":"test",         "version":1,         "timestamp":"2015-01-16T15:53:02.799Z",         "size":5359}]}}

Stack Trace:
java.lang.AssertionError: Could not successfully add blob after 150 attempts. Expecting 2 items. time elapsed 16.021  output  for url is {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "response":{
    "numFound":1,
    "start":0,
    "docs":[{
        "id":"test/1",
        "md5":"33641e30a627df1efbdb4386e71b05e0",
        "blobName":"test",
        "version":1,
        "timestamp":"2015-01-16T15:53:02.799Z",
        "size":5359}]}}
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestBlobHandler.postAndCheck(TestBlobHandler.java:150)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:114)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1421423713058,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1421423713058,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1421423713058,generation=3,filelist=[_5.cfe, _5.cfs, _5.si, _9.cfe, _9.cfs, _9.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1421423713058,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1421423713058,generation=2,filelist=[_5.cfe, _5.cfs, _5.si, _7.cfe, _7.cfs, _7.si, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1421423713058,generation=3,filelist=[_5.cfe, _5.cfs, _5.si, _9.cfe, _9.cfs, _9.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:1368CA940862CE7B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1139)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63652/eui/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63652/eui/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63652/eui/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 46 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:54568/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:54568/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:54568/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:20238/w_/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20238/w_/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 47 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62819/m_/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62819/m_/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([36BFD1A4782AC078:B7595FBC0F75A044]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62819/m_/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9892 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/init-core-data-001
   [junit4]   2> 523540 T1938 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /m_/
   [junit4]   2> 523546 T1938 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 523548 T1938 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 523548 T1939 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 523648 T1938 oasc.ZkTestServer.run start zk server on port:62807
   [junit4]   2> 523649 T1938 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 523650 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 523654 T1946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a22ff84 name:ZooKeeperConnection Watcher:127.0.0.1:62807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 523655 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 523655 T1938 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 523656 T1938 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 523659 T1938 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 523660 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 523661 T1949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13f108c4 name:ZooKeeperConnection Watcher:127.0.0.1:62807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 523661 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 523662 T1938 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 523662 T1938 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 523664 T1938 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 523666 T1938 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 523667 T1938 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 523669 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 523670 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 523673 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 523674 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 523676 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 523677 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 523679 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 523680 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 523682 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 523682 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 523684 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 523685 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 523687 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 523688 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 523690 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 523690 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 523692 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 523693 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 523695 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 523696 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 523698 T1938 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 523698 T1938 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 525483 T1938 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 525490 T1938 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62820
   [junit4]   2> 525490 T1938 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 525491 T1938 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 525491 T1938 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002
   [junit4]   2> 525492 T1938 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002/'
   [junit4]   2> 525565 T1938 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002/solr.xml
   [junit4]   2> 525692 T1938 oasc.CoreContainer.<init> New CoreContainer 1526381141
   [junit4]   2> 525692 T1938 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002/]
   [junit4]   2> 525694 T1938 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 525694 T1938 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 525695 T1938 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 525695 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 525695 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 525696 T1938 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 525696 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 525697 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 525697 T1938 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 525697 T1938 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 525698 T1938 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 525699 T1938 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 525699 T1938 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 525699 T1938 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 525700 T1938 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 525700 T1938 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62807/solr
   [junit4]   2> 525701 T1938 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 525701 T1938 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 525702 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 525704 T1961 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4eb3255a name:ZooKeeperConnection Watcher:127.0.0.1:62807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 525704 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 525705 T1938 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 525707 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 525709 T1964 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60fcbf5d name:ZooKeeperConnection Watcher:127.0.0.1:62807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 525709 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 525711 T1938 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 525713 T1938 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 525715 T1938 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 525716 T1938 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 525718 T1938 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 525720 T1938 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 525721 T1938 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62819_m_
   [junit4]   2> 525722 T1938 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62819_m_
   [junit4]   2> 525724 T1938 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 525725 T1938 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 525727 T1938 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 525728 T1938 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62819_m_
   [junit4]   2> 525729 T1938 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 525730 T1938 oasc.Overseer.start Overseer (id=93154312179613699-127.0.0.1:62819_m_-n_0000000000) starting
   [junit4]   2> 525732 T1938 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 525737 T1938 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 525738 T1966 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 525738 T1938 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 525740 T1938 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 525741 T1938 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 525744 T1965 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 525750 T1968 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 525750 T1968 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 525751 T1964 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 525751 T1968 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 525753 T1965 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62819/m_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:62819_m_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 525754 T1965 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62819/m_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:62819_m_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 525754 T1965 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 525755 T1965 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 525756 T1964 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 526752 T1968 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 526753 T1968 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 526753 T1968 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 526754 T1968 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 526754 T1968 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 526754 T1968 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002/collection1/'
   [junit4]   2> 526756 T1968 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%2036BFD1A4782AC078-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 526757 T1968 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%2036BFD1A4782AC078-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 526757 T1968 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%2036BFD1A4782AC078-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 526795 T1968 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 526797 T1968 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 526810 T1968 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 526830 T1968 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 526873 T1968 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 526874 T1968 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 526892 T1968 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 527211 T1968 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 527224 T1968 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 527227 T1968 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 527235 T1968 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 527244 T1968 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 527248 T1968 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 527249 T1968 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 527249 T1968 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 527249 T1968 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 527249 T1968 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 527250 T1968 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 527250 T1968 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 527250 T1968 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 527250 T1968 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-001/control/data/
   [junit4]   2> 527251 T1968 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@392b00ce
   [junit4]   2> 527251 T1968 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-001/control/data
   [junit4]   2> 527251 T1968 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-001/control/data/index/
   [junit4]   2> 527252 T1968 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 527252 T1968 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-001/control/data/index
   [junit4]   2> 527252 T1968 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 527253 T1968 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4215f26a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ede2f0b),segFN=segments_1,generation=1}
   [junit4]   2> 527253 T1968 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 527256 T1968 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 527256 T1968 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 527257 T1968 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 527257 T1968 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 527257 T1968 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 527257 T1968 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 527258 T1968 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 527258 T1968 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 527258 T1968 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 527259 T1968 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 527259 T1968 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 527259 T1968 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 527260 T1968 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 527260 T1968 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 527260 T1968 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 527261 T1968 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 527261 T1968 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 527261 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 527262 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 527262 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 527263 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 527263 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 527263 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 527264 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 527264 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 527264 T1968 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 527265 T1968 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 527286 T1968 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 527288 T1968 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 527290 T1968 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 527291 T1968 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 527293 T1968 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 527296 T1968 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 527297 T1968 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 527297 T1968 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=75.80859375, floorSegmentMB=0.5986328125, forceMergeDeletesPctAllowed=21.55119951869103, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1816737238506173
   [junit4]   2> 527298 T1968 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4215f26a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ede2f0b),segFN=segments_1,generation=1}
   [junit4]   2> 527298 T1968 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 527298 T1968 oass.SolrIndexSearcher.<init> Opening Searcher@71426ef8[collection1] main
   [junit4]   2> 527299 T1968 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 527299 T1968 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 527299 T1968 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 527300 T1968 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 527300 T1968 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 527300 T1968 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 527300 T1968 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 527301 T1968 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 527301 T1968 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 527301 T1968 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 527302 T1968 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 527302 T1969 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71426ef8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 527302 T1972 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62819/m_ collection:control_collection shard:shard1
   [junit4]   2> 527302 T1938 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 527303 T1938 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 527303 T1972 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 527306 T1938 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 527306 T1972 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 527306 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 527308 T1964 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 527308 T1972 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 527308 T1976 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6305e629 name:ZooKeeperConnection Watcher:127.0.0.1:62807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 527308 T1972 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 527308 T1965 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 527308 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C213 name=collection1 org.apache.solr.core.SolrCore@2ab769bc url=http://127.0.0.1:62819/m_/collection1 node=127.0.0.1:62819_m_ C213_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:62819/m_, node_name=127.0.0.1:62819_m_, core=collection1}
   [junit4]   2> 527308 T1972 C213 P62819 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62819/m_/collection1/
   [junit4]   2> 527309 T1938 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 527309 T1972 C213 P62819 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 527309 T1964 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 527310 T1938 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 527310 T1972 C213 P62819 oasc.SyncStrategy.syncToMe http://127.0.0.1:62819/m_/collection1/ has no replicas
   [junit4]   2> 527310 T1972 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62819/m_/collection1/ shard1
   [junit4]   2> 527310 T1972 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 527312 T1938 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 527313 T1977 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 527314 T1965 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62819/m_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 527416 T1976 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 527466 T1972 oasc.ZkController.register We are http://127.0.0.1:62819/m_/collection1/ and leader is http://127.0.0.1:62819/m_/collection1/
   [junit4]   2> 527467 T1972 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62819/m_
   [junit4]   2> 527467 T1972 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 527467 T1972 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 527467 T1972 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 527468 T1964 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 527469 T1965 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62819/m_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:62819_m_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 527470 T1965 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62819/m_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:62819_m_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 527572 T1964 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 527572 T1976 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 529317 T1938 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 529318 T1938 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 529322 T1938 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62855
   [junit4]   2> 529323 T1938 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 529323 T1938 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 529324 T1938 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-003
   [junit4]   2> 529324 T1938 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-003/'
   [junit4]   2> 529390 T1938 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-003/solr.xml
   [junit4]   2> 529494 T1938 oasc.CoreContainer.<init> New CoreContainer 723531293
   [junit4]   2> 529495 T1938 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 36BFD1A4782AC078-001/tempDir-003/]
   [junit4]   2> 529496 T1938 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 529497 T1938 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 529497 T1938 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 529497 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 529498 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 529498 T1938 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 529499 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 529499 T1938 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 529499 T1938 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 529500 T1938 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 529500 T1938 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 529501 T1938 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 529501 T1938 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 529502 T1938 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 529502 T1938 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 529502 T1938 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62807/solr
   [junit4]   2> 529503 T1938 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 529503 T1938 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 529504 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 529506 T1989 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f494ae1 name:ZooKeeperConnection Watcher:127.0.0.1:62807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 529506 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 529507 T1938 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 529509 T1938 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 529511 T1992 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31152078 name:ZooKeeperConnection Watcher:127.0.0.1:62807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 529511 T1938 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 529515 T1938 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 530519 T1938 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62854_m_
   [junit4]   2> 530520 T1938 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62854_m_
   [junit4]   2> 530523 T1938 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 530531 T1993 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 530531 T1993 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 530532 T1964 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 530532 T1993 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 530533 T1965 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null

[...truncated too long message...]

ation params={version=2&slave=false&qt=/replication&command=details&wt=javabin} status=0 QTime=0 
   [junit4]   2> 2256713 T3451 C2584 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={version=2&qt=/replication&command=details&wt=javabin&_trace=getDetails} status=0 QTime=3 
   [junit4]   2> 2256713 T2583 oas.SolrTestCaseJ4.tearDown ###Ending doTestSnapPullWithMasterUrl
   [junit4]   2> 2256714 T2583 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1979538987
   [junit4]   2> 2256714 T2583 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@64c2f394
   [junit4]   2> 2256715 T2583 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=497,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 2256715 T2583 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2256716 T2583 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2256716 T2583 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2256717 T2583 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2256717 T2583 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2256718 T2583 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data/index.20150116075532469 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data/index.20150116075532469;done=false>>]
   [junit4]   2> 2256718 T2583 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data/index.20150116075532469
   [junit4]   2> 2256718 T2583 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 2256719 T2583 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-001/collection1/data
   [junit4]   2> 2256720 T2583 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2256915 T2583 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=122898781
   [junit4]   2> 2256916 T2583 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4fd6b5ec
   [junit4]   2> 2256916 T2583 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 2256917 T2583 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2256918 T2583 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2256918 T2583 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2256919 T2583 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2256919 T2583 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2256920 T2583 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data/index.20150116075532129 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data/index.20150116075532129;done=false>>]
   [junit4]   2> 2256920 T2583 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data/index.20150116075532129
   [junit4]   2> 2256920 T2583 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data;done=false>>]
   [junit4]   2> 2256921 T2583 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001/solr-instance-002/collection1/data
   [junit4]   2> 2256922 T2583 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2257134 T2583 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 36BFD1A4782AC078-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=hr_HR, timezone=Mexico/BajaNorte
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=211883560,total=286261248
   [junit4]   2> NOTE: All tests run in this JVM: [TestFoldingMultitermQuery, CacheHeaderTest, LeaderElectionIntegrationTest, CurrencyFieldOpenExchangeTest, TestCSVResponseWriter, TestCloudSchemaless, TestMissingGroups, ResponseHeaderTest, TestLazyCores, MoreLikeThisHandlerTest, DistribCursorPagingTest, TestDynamicFieldCollectionResource, HdfsSyncSliceTest, ExternalCollectionsTest, DirectUpdateHandlerOptimizeTest, TestHashPartitioner, OutOfBoxZkACLAndCredentialsProvidersTest, TestCollapseQParserPlugin, TestPHPSerializedResponseWriter, TestManagedSynonymFilterFactory, TestDefaultStatsCache, PrimUtilsTest, TestUniqueKeyFieldResource, PluginInfoTest, TestLeaderElectionZkExpiry, HardAutoCommitTest, TestComponentsName, SolrIndexSplitterTest, AlternateDirectoryTest, ClusterStateTest, TestSolrQueryParserResource, TestReversedWildcardFilterFactory, TestSchemaVersionResource, PeerSyncTest, TestLMDirichletSimilarityFactory, SolrCoreCheckLockOnStartupTest, VMParamsZkACLAndCredentialsProvidersTest, BasicDistributedZk2Test, HdfsBasicDistributedZkTest, TestReload, LeaderFailoverAfterPartitionTest, SyncSliceTest, TestFieldTypeResource, ScriptEngineTest, SolrRequestParserTest, TestComplexPhraseQParserPlugin, ReturnFieldsTest, TestDefaultSimilarityFactory, EchoParamsTest, MultiTermTest, TestRawResponseWriter, TestConfigOverlay, TestRandomDVFaceting, SynonymTokenizerTest, AddBlockUpdateTest, TestReplicationHandler]
   [junit4] Completed on J2 in 724.62s, 13 tests, 1 failure <<< FAILURES!

[...truncated 213 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1872 tests, 6 errors, 2 failures, 57 ignored (9 assumptions)

Total time: 143 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 625686743 bytes
Compression is 0.0%
Took 2 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 735 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/735/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
file handle leaks: [SeekableByteChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler B77BCC13B9C41EA5-001/index-SimpleFSDirectory-043/replication.properties)]

Stack Trace:
java.lang.RuntimeException: file handle leaks: [SeekableByteChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler B77BCC13B9C41EA5-001/index-SimpleFSDirectory-043/replication.properties)]
	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5]:0)
	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:182)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.Exception
	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:259)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:214)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:231)
	at java.nio.file.Files.newByteChannel(Files.java:317)
	at java.nio.file.Files.newByteChannel(Files.java:363)
	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:61)
	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2474)
	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:654)
	at org.apache.solr.handler.ReplicationHandler.loadReplicationProperties(ReplicationHandler.java:832)
	at org.apache.solr.handler.SnapPuller.logReplicationTimeAndConfFiles(SnapPuller.java:571)
	at org.apache.solr.handler.SnapPuller.fetchLatestIndex(SnapPuller.java:510)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:343)
	at org.apache.solr.handler.SnapPuller$1.run(SnapPuller.java:224)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19959/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19959/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5:369D420BCE9B7E99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19959/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61134/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61134/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5:369D420BCE9B7E99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61134/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:12808/u_l/wz/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12808/u_l/wz/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5:369D420BCE9B7E99]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61619/vuj/ra/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61619/vuj/ra/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5:369D420BCE9B7E99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor42.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61619/vuj/ra/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 46 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more




Build Log:
[...truncated 9822 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/init-core-data-001
   [junit4]   2> 290269 T1459 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 290275 T1459 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 290277 T1459 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 290277 T1460 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 290377 T1459 oasc.ZkTestServer.run start zk server on port:19933
   [junit4]   2> 290378 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 290379 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 290383 T1467 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@192797e3 name:ZooKeeperConnection Watcher:127.0.0.1:19933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 290383 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 290384 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 290384 T1459 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 290403 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 290404 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 290405 T1470 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64e6014f name:ZooKeeperConnection Watcher:127.0.0.1:19933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 290406 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 290406 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 290406 T1459 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 290408 T1459 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 290409 T1459 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 290410 T1459 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 290411 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 290412 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 290413 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 290414 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 290516 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 290516 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 290518 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 290518 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 290520 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 290520 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 290522 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 290522 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 290523 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 290524 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 290525 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 290526 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 290527 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 290527 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 290529 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 290529 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 290531 T1459 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 290531 T1459 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 291657 T1459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 291663 T1459 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19947
   [junit4]   2> 291663 T1459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 291664 T1459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 291664 T1459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002
   [junit4]   2> 291664 T1459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002/'
   [junit4]   2> 291701 T1459 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002/solr.xml
   [junit4]   2> 291767 T1459 oasc.CoreContainer.<init> New CoreContainer 763427236
   [junit4]   2> 291768 T1459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002/]
   [junit4]   2> 291769 T1459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 291769 T1459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 291769 T1459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 291770 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 291770 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 291770 T1459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 291770 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 291771 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 291771 T1459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 291771 T1459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 291771 T1459 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 291772 T1459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 291773 T1459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 291773 T1459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 291773 T1459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 291773 T1459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19933/solr
   [junit4]   2> 291774 T1459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 291774 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 291775 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 291777 T1482 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@259015b name:ZooKeeperConnection Watcher:127.0.0.1:19933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 291777 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 291778 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 291780 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 291782 T1485 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16d9d277 name:ZooKeeperConnection Watcher:127.0.0.1:19933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 291782 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 291784 T1459 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 291786 T1459 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 291788 T1459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 291789 T1459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 291791 T1459 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 291793 T1459 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 291794 T1459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19946_
   [junit4]   2> 291795 T1459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19946_
   [junit4]   2> 291796 T1459 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 291797 T1459 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 291799 T1459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 291800 T1459 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:19946_
   [junit4]   2> 291800 T1459 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 291802 T1459 oasc.Overseer.start Overseer (id=93149131939840003-127.0.0.1:19946_-n_0000000000) starting
   [junit4]   2> 291803 T1459 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 291808 T1459 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 291809 T1487 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 291809 T1459 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 291812 T1459 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 291813 T1459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 291815 T1486 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 291820 T1489 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 291820 T1489 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 291821 T1485 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 291821 T1489 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 291823 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19946",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19946_",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 291823 T1486 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19946",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19946_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 291824 T1486 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 291825 T1486 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 291826 T1485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 292821 T1489 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 292822 T1489 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 292822 T1489 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 292823 T1489 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 292823 T1489 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 292824 T1489 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002/collection1/'
   [junit4]   2> 292825 T1489 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 292826 T1489 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 292826 T1489 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 292870 T1489 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 292873 T1489 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 292889 T1489 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 292917 T1489 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 292973 T1489 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 293075 T1489 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 293086 T1489 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 293884 T1489 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 293900 T1489 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 293903 T1489 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 293917 T1489 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 293922 T1489 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 293927 T1489 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 293928 T1489 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 293928 T1489 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 293929 T1489 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 293929 T1489 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 293930 T1489 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 293930 T1489 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 293930 T1489 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 293931 T1489 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/control/data/
   [junit4]   2> 293931 T1489 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@47fca282
   [junit4]   2> 293932 T1489 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/control/data
   [junit4]   2> 293932 T1489 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/control/data/index/
   [junit4]   2> 293932 T1489 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 293942 T1489 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/control/data/index
   [junit4]   2> 293943 T1489 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=65.0927734375, floorSegmentMB=0.931640625, forceMergeDeletesPctAllowed=2.4024325619411844, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 293962 T1489 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/index-NIOFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@267e7a41; maxCacheMB=0.3544921875 maxMergeSizeMB=0.3984375)),segFN=segments_1,generation=1}
   [junit4]   2> 293962 T1489 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 293968 T1489 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 293969 T1489 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 293969 T1489 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 293969 T1489 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 293970 T1489 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 293970 T1489 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 293970 T1489 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 293971 T1489 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 293971 T1489 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 293972 T1489 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 293973 T1489 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 293973 T1489 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 293974 T1489 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 293974 T1489 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 293975 T1489 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 293975 T1489 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 293976 T1489 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 293976 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 293977 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 293978 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 293978 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 293979 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 293979 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 293980 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 293980 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 293981 T1489 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 293981 T1489 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 294003 T1489 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 294006 T1489 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 294008 T1489 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 294010 T1489 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 294012 T1489 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 294017 T1489 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 294017 T1489 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 294018 T1489 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=16.166015625, floorSegmentMB=1.470703125, forceMergeDeletesPctAllowed=9.599307316236574, segmentsPerTier=11.0, maxCFSSegmentSizeMB=0.49609375, noCFSRatio=0.6559020825501003
   [junit4]   2> 294040 T1489 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/index-NIOFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@267e7a41; maxCacheMB=0.3544921875 maxMergeSizeMB=0.3984375)),segFN=segments_1,generation=1}
   [junit4]   2> 294041 T1489 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 294041 T1489 oass.SolrIndexSearcher.<init> Opening Searcher@64e82209[collection1] main
   [junit4]   2> 294042 T1489 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 294043 T1489 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 294043 T1489 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 294043 T1489 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 294043 T1489 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 294044 T1489 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 294044 T1489 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 294045 T1489 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 294045 T1489 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 294045 T1489 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 294047 T1490 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@64e82209[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 294047 T1489 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 294048 T1493 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19946 collection:control_collection shard:shard1
   [junit4]   2> 294049 T1459 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 294049 T1459 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 294049 T1493 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 294054 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 294055 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 294055 T1493 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 294057 T1497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d41f09c name:ZooKeeperConnection Watcher:127.0.0.1:19933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 294057 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 294058 T1485 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 294058 T1493 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 294058 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 294058 T1493 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 294059 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C277 name=collection1 org.apache.solr.core.SolrCore@6b90e6f9 url=http://127.0.0.1:19946/collection1 node=127.0.0.1:19946_ C277_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:19946, node_name=127.0.0.1:19946_, state=down}
   [junit4]   2> 294060 T1493 C277 P19946 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19946/collection1/
   [junit4]   2> 294060 T1459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 294060 T1493 C277 P19946 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 294061 T1493 C277 P19946 oasc.SyncStrategy.syncToMe http://127.0.0.1:19946/collection1/ has no replicas
   [junit4]   2> 294061 T1493 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19946/collection1/ shard1
   [junit4]   2> 294061 T1493 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 294061 T1497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 294061 T1485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 294064 T1459 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 294065 T1459 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 294066 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 294067 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 294067 T1500 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 294074 T1501 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78295f4a name:ZooKeeperConnection Watcher:127.0.0.1:19933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 294074 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19946",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 294074 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 294075 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 294078 T1500 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 294080 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 2
   [junit4]   2> 294081 T1486 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 294082 T1497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 294187 T1486 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 294188 T1497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 294221 T1493 oasc.ZkController.register We are http://127.0.0.1:19946/collection1/ and leader is http://127.0.0.1:19946/collection1/
   [junit4]   2> 294222 T1493 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:19946
   [junit4]   2> 294222 T1493 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 294222 T1493 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 294222 T1493 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 294224 T1485 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 294226 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19946",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19946_",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 294227 T1486 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19946",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19946_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 294330 T1485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 294330 T1497 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 295998 T1459 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 295999 T1459 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 296002 T1459 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19960
   [junit4]   2> 296003 T1459 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 296004 T1459 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 296004 T1459 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003
   [junit4]   2> 296004 T1459 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003/'
   [junit4]   2> 296069 T1459 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003/solr.xml
   [junit4]   2> 296185 T1459 oasc.CoreContainer.<init> New CoreContainer 1689421950
   [junit4]   2> 296186 T1459 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003/]
   [junit4]   2> 296187 T1459 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 296187 T1459 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 296188 T1459 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 296188 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 296188 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 296189 T1459 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 296189 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 296189 T1459 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 296190 T1459 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 296190 T1459 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 296190 T1459 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 296191 T1459 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 296192 T1459 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 296192 T1459 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 296192 T1459 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 296193 T1459 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19933/solr
   [junit4]   2> 296193 T1459 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 296193 T1459 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 296194 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 296196 T1513 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a7df9cb name:ZooKeeperConnection Watcher:127.0.0.1:19933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 296197 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 296198 T1459 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 296201 T1459 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 296203 T1516 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5330ef5c name:ZooKeeperConnection Watcher:127.0.0.1:19933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 296203 T1459 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 296208 T1459 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 297213 T1459 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19959_
   [junit4]   2> 297214 T1459 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19959_
   [junit4]   2> 297217 T1459 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 297225 T1517 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 297225 T1517 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 297227 T1485 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 297227 T1517 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 297227 T1517 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 297228 T1486 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19959",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19959_",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 297228 T1517 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 297228 T1486 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19959",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19959_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 297229 T1486 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 297229 T1486 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 297239 T1517 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 297240 T1517 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 297240 T1486 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 297241 T1516 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 297242 T1516 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 298240 T1517 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 298241 T1517 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 298241 T1517 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 298242 T1517 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 298242 T1517 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298242 T1517 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003/collection1/'
   [junit4]   2> 298245 T1517 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 298245 T1517 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 298246 T1517 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest%20B77BCC13B9C41EA5-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 298292 T1517 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 298295 T1517 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 298311 T1517 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 298338 T1517 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 298402 T1517 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 298504 T1517 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 298516 T1517 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 298977 T1517 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 298994 T1517 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 298997 T1517 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 299009 T1517 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 299014 T1517 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 299019 T1517 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 299020 T1517 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 299021 T1517 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 299021 T1517 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 299022 T1517 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 299022 T1517 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 299022 T1517 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 299023 T1517 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 299023 T1517 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/jetty1/
   [junit4]   2> 299023 T1517 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@47fca282
   [junit4]   2> 299024 T1517 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/jetty1
   [junit4]   2> 299024 T1517 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/jetty1/index/
   [junit4]   2> 299024 T1517 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 299027 T1517 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/tempDir-001/jetty1/index
   [junit4]   2> 299028 T1517 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=65.0927734375, floorSegmentMB=0.931640625, forceMergeDeletesPctAllowed=2.4024325619411844, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 299037 T1517 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest B77BCC13B9C41EA5-001/index-NIOFSDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@267e7a41; maxCacheMB=0.3544921875 maxMergeSizeMB=0.3984375)),segFN=segments_1,generation=1}
   [junit4]   2> 299037 T1517 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 299045 T1517 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 299045 T1517 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 299045 T1517 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 299046 T1517 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 299046 T1517 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 299046 T1517 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 299047 T1517 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 299047 T1517 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 299047 T1517 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 299048 T1517 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 299049 T1517 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 299049 T1517 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 299050 T1517 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 299050 T1517 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 299051 T1517 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 299054 T1517 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 299055 T1517 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 299055 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 299056 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 299057 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 299057 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 299058 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 299059 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 299059 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 299060 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 299060 T1517 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 299061 T1517 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 299088 T1517 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 299090 T1517 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 299092 T1517 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 299095 T1517 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 299097 T1517 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 299100 T1517 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 299101 T1517 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 299102 T1517 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=16.166015625, floorSegmentMB=1.470703125, forceMergeDeletesPctAllowed=9.599307316236574, segmentsPerTier=11.0, maxCFSSegmentSizeMB=0.49609375, noCFSRatio=0.6559020825501003
   [junit4]   2> 299107 T1517 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(NIOFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/

[...truncated too long message...]

3B9C41EA5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ms -Dtests.timezone=America/Guayaquil -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   71.1s J0 | ReplicationFactorTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61619/vuj/ra/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B77BCC13B9C41EA5:369D420BCE9B7E99]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61619/vuj/ra/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
   [junit4]    > 	... 46 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 51 more
   [junit4]   2> 2312573 T17132 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest B77BCC13B9C41EA5-001
   [junit4]   2> 71055 T17131 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ms, timezone=America/Guayaquil
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=98873800,total=457703424
   [junit4]   2> NOTE: All tests run in this JVM: [SolrCmdDistributorTest, SearchHandlerTest, TriLevelCompositeIdRoutingTest, TestUtils, TestIBSimilarityFactory, CustomCollectionTest, HttpPartitionTest, SolrTestCaseJ4Test, TestRTGBase, TestDFRSimilarityFactory, TestSearcherReuse, TestSort, DeleteReplicaTest, ShowFileRequestHandlerTest, StatelessScriptUpdateProcessorFactoryTest, SampleTest, RegexBoostProcessorTest, DistanceFunctionTest, ZkSolrClientTest, RequiredFieldsTest, TestCodecSupport, TestFieldCollectionResource, TestBlendedInfixSuggestions, TestCollationField, TestDocBasedVersionConstraints, RecoveryZkTest, TestPartialUpdateDeduplication, TestConfigSets, TestStressReorder, TestLuceneMatchVersion, TestMaxScoreQueryParser, TestExceedMaxTermLength, TestNonDefinedSimilarityFactory, TestLFUCache, TestFieldTypeCollectionResource, OverseerTest, TestInitQParser, DistributedQueueTest, DistributedIntervalFacetingTest, TestShardHandlerFactory, TestNRTOpen, MBeansHandlerTest, TestSuggestSpellingConverter, TestSolr4Spatial2, SimpleCollectionCreateDeleteTest, SignatureUpdateProcessorFactoryTest, TestAnalyzeInfixSuggestions, TestFuzzyAnalyzedSuggestions, TestReplicaProperties, CoreAdminRequestStatusTest, FastVectorHighlighterTest, UpdateParamsTest, SpellCheckCollatorTest, TestCoreContainer, SimpleFacetsTest, SuggesterTest, HdfsChaosMonkeySafeLeaderTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, ShardRoutingTest, BasicZkTest, DistributedSpellCheckComponentTest, TestStressVersions, SolrCoreTest, BasicFunctionalityTest, TestLazyCores, OverseerCollectionProcessorTest, HighlighterTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, CurrencyFieldXmlFileTest, TestCoreDiscovery, CoreAdminHandlerTest, TestTrie, PolyFieldTest, SchemaVersionSpecificBehaviorTest, TestUpdate, TestAtomicUpdateErrorCases, QueryEqualityTest, XsltUpdateRequestHandlerTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, TestWriterPerf, PrimitiveFieldTypeTest, TermsComponentTest, RequestHandlersTest, TestValueSourceCache, TestIndexingPerformance, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, QueryParsingTest, TestPHPSerializedResponseWriter, TestComponentsName, HighlighterConfigTest, TestQuerySenderListener, TestQuerySenderNoQuery, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, TestDocSet, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, TestJmxMonitoredMap, TimeZoneUtilsTest, ScriptEngineTest, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, PreAnalyzedFieldTest, PrimUtilsTest, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, SliceStateTest, UUIDFieldTest, FileUtilsTest, CircularListTest, CursorPagingTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteInactiveReplicaTest, MultiThreadedOCPTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 71.33s, 1 test, 1 error <<< FAILURES!

[...truncated 236 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1872 tests, 1 suite-level error, 4 errors, 57 ignored (9 assumptions)

Total time: 152 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 727963180 bytes
Compression is 0.0%
Took 2 min 2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 734 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/734/

7 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
ERROR: SolrIndexSearcher opens=8 closes=7

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:442)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:188)
	at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=5844, name=qtp1510795348-5844, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:503)         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:228)         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:773)         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1770)         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2006)         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:781)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:417)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)         at org.eclipse.jetty.server.Server.handle(Server.java:368)         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=5854, name=searcherExecutor-2407-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=5844, name=qtp1510795348-5844, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:503)
        at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
        at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
        at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:228)
        at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:773)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1770)
        at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2006)
        at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:781)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:417)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
        at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
        at org.eclipse.jetty.server.Server.handle(Server.java:368)
        at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
        at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
        at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
        at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=5854, name=searcherExecutor-2407-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07]:0)


REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Captured an uncaught exception in thread: Thread[id=2625, name=Thread-961, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2625, name=Thread-961, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:17240: Could not find collection : awholynewstresscollection_collection3_7
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:928)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21140/ea_/g/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21140/ea_/g/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07:98C88590B9A69F3B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21140/ea_/g/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49927/xd_dhr/s/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49927/xd_dhr/s/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07:98C88590B9A69F3B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49927/xd_dhr/s/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:31570/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31570/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07:98C88590B9A69F3B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44984/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44984/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07:98C88590B9A69F3B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44984/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9737 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderFailoverAfterPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/init-core-data-001
   [junit4]   2> 134779 T445 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /xd_dhr/s
   [junit4]   2> 134788 T445 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 134790 T445 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 134791 T446 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 134890 T445 oasc.ZkTestServer.run start zk server on port:49889
   [junit4]   2> 134892 T445 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 134893 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134898 T453 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ce62202 name:ZooKeeperConnection Watcher:127.0.0.1:49889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134898 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134899 T445 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 134900 T445 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 134903 T445 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 134905 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134907 T456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35e47b48 name:ZooKeeperConnection Watcher:127.0.0.1:49889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134907 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134908 T445 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 134908 T445 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 134911 T445 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 134913 T445 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 134915 T445 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 134917 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 134918 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 134922 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 134923 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 134926 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 134927 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 134929 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 134930 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 134933 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 134933 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 134936 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 134937 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 134939 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 134940 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 134943 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 134944 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 134953 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 134954 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 134957 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 134957 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 134960 T445 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 134961 T445 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 136521 T445 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 136527 T445 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49898
   [junit4]   2> 136528 T445 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 136529 T445 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 136529 T445 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002
   [junit4]   2> 136530 T445 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002/'
   [junit4]   2> 136578 T445 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002/solr.xml
   [junit4]   2> 136666 T445 oasc.CoreContainer.<init> New CoreContainer 1047472324
   [junit4]   2> 136667 T445 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002/]
   [junit4]   2> 136669 T445 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 136670 T445 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 136670 T445 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 136671 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 136671 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 136671 T445 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 136672 T445 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 136672 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 136673 T445 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 136673 T445 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 136674 T445 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 136675 T445 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 136675 T445 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 136676 T445 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 136676 T445 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49889/solr
   [junit4]   2> 136677 T445 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 136677 T445 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136678 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136681 T468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20bd86c2 name:ZooKeeperConnection Watcher:127.0.0.1:49889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136681 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136683 T445 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136685 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136687 T471 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2684bd02 name:ZooKeeperConnection Watcher:127.0.0.1:49889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136688 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136690 T445 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 136693 T445 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 136695 T445 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 136697 T445 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 136700 T445 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 136702 T445 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 136703 T445 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49897_xd_dhr%2Fs
   [junit4]   2> 136704 T445 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49897_xd_dhr%2Fs
   [junit4]   2> 136707 T445 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 136708 T445 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 136710 T445 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 136712 T445 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49897_xd_dhr%2Fs
   [junit4]   2> 136712 T445 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 136714 T445 oasc.Overseer.start Overseer (id=93142989196427267-127.0.0.1:49897_xd_dhr%2Fs-n_0000000000) starting
   [junit4]   2> 136716 T445 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 136723 T445 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 136724 T473 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 136724 T445 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 136726 T445 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 136728 T445 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 136731 T472 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 136738 T475 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 136739 T475 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 136740 T475 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 136740 T471 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136743 T472 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49897/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49897_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 136744 T472 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49897/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49897_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 136744 T472 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 136745 T472 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 136746 T471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 137740 T475 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 137741 T475 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 137742 T475 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 137742 T475 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 137743 T475 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 137743 T475 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002/collection1/'
   [junit4]   2> 137746 T475 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 137747 T475 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 137748 T475 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 137793 T475 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 137796 T475 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 137835 T475 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 137863 T475 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 137922 T475 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 137923 T475 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 137942 T475 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 138416 T475 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 138434 T475 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 138438 T475 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 138462 T475 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 138468 T475 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 138474 T475 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 138475 T475 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 138476 T475 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 138476 T475 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 138477 T475 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 138477 T475 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 138478 T475 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 138478 T475 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 138479 T475 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-001/control/data/
   [junit4]   2> 138479 T475 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5025249a
   [junit4]   2> 138480 T475 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-001/control/data
   [junit4]   2> 138481 T475 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-001/control/data/index/
   [junit4]   2> 138481 T475 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 138482 T475 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-001/control/data/index
   [junit4]   2> 138482 T475 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=36.310546875, floorSegmentMB=0.701171875, forceMergeDeletesPctAllowed=12.97022699083391, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 138484 T475 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e84e7fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@379adc24),segFN=segments_1,generation=1}
   [junit4]   2> 138484 T475 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 138491 T475 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 138492 T475 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 138492 T475 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 138493 T475 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 138493 T475 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 138493 T475 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 138494 T475 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 138495 T475 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 138495 T475 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 138496 T475 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 138497 T475 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 138497 T475 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 138498 T475 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 138499 T475 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 138499 T475 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 138500 T475 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 138501 T475 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 138502 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 138503 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 138504 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 138505 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 138505 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 138506 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 138507 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 138508 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 138508 T475 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 138509 T475 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 138541 T475 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 138545 T475 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 138547 T475 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 138550 T475 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 138554 T475 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 138560 T475 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 138560 T475 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 138561 T475 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3034171788423752]
   [junit4]   2> 138563 T475 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e84e7fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@379adc24),segFN=segments_1,generation=1}
   [junit4]   2> 138563 T475 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 138564 T475 oass.SolrIndexSearcher.<init> Opening Searcher@1faf2ba6[collection1] main
   [junit4]   2> 138564 T475 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 138565 T475 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 138566 T475 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 138566 T475 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 138567 T475 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 138567 T475 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 138568 T475 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 138568 T475 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 138569 T475 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 138569 T475 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 138571 T476 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1faf2ba6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 138571 T475 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 138573 T479 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49897/xd_dhr/s collection:control_collection shard:shard1
   [junit4]   2> 138573 T445 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 138574 T479 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 138574 T445 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 138580 T445 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 138580 T479 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 138581 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 138583 T471 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 138584 T479 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 138583 T483 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c77ee25 name:ZooKeeperConnection Watcher:127.0.0.1:49889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 138584 T479 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 138584 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 138584 T472 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C24 name=collection1 org.apache.solr.core.SolrCore@6cf58ce5 url=http://127.0.0.1:49897/xd_dhr/s/collection1 node=127.0.0.1:49897_xd_dhr%2Fs C24_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:49897/xd_dhr/s, node_name=127.0.0.1:49897_xd_dhr%2Fs, core=collection1, state=down}
   [junit4]   2> 138585 T479 C24 P49897 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49897/xd_dhr/s/collection1/
   [junit4]   2> 138586 T445 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 138586 T479 C24 P49897 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 138587 T479 C24 P49897 oasc.SyncStrategy.syncToMe http://127.0.0.1:49897/xd_dhr/s/collection1/ has no replicas
   [junit4]   2> 138588 T471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138588 T445 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 138588 T479 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49897/xd_dhr/s/collection1/ shard1
   [junit4]   2> 138589 T479 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 138591 T445 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 138594 T484 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 138596 T472 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49897/xd_dhr/s",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 138699 T483 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138747 T479 oasc.ZkController.register We are http://127.0.0.1:49897/xd_dhr/s/collection1/ and leader is http://127.0.0.1:49897/xd_dhr/s/collection1/
   [junit4]   2> 138748 T479 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49897/xd_dhr/s
   [junit4]   2> 138748 T479 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 138748 T479 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 138749 T479 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 138750 T471 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 138752 T472 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49897/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49897_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 138752 T472 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49897/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49897_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 138856 T471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138856 T483 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 140302 T445 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 140304 T445 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 140307 T445 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49910
   [junit4]   2> 140308 T445 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 140309 T445 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 140309 T445 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003
   [junit4]   2> 140310 T445 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003/'
   [junit4]   2> 140357 T445 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003/solr.xml
   [junit4]   2> 140444 T445 oasc.CoreContainer.<init> New CoreContainer 1113411387
   [junit4]   2> 140445 T445 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003/]
   [junit4]   2> 140446 T445 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 140447 T445 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 140447 T445 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 140448 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 140448 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 140449 T445 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 140449 T445 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 140449 T445 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 140450 T445 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 140450 T445 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 140451 T445 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 140452 T445 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 140452 T445 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 140453 T445 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 140453 T445 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49889/solr
   [junit4]   2> 140454 T445 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 140454 T445 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 140455 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140457 T496 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f78284 name:ZooKeeperConnection Watcher:127.0.0.1:49889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140458 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140459 T445 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 140462 T445 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140464 T499 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2bb1d0aa name:ZooKeeperConnection Watcher:127.0.0.1:49889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140464 T445 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140470 T445 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 141474 T445 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49909_xd_dhr%2Fs
   [junit4]   2> 141475 T445 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49909_xd_dhr%2Fs
   [junit4]   2> 141478 T445 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 141486 T500 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 141487 T500 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 141488 T500 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 141488 T471 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 141489 T472 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49909/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49909_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 141490 T472 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49909/xd_dhr/s",
   [junit4]   2> 	  "node_name":"127.0.0.1:49909_xd_dhr%2Fs",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 141491 T472 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 141491 T472 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 141493 T499 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 141493 T483 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 141493 T471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 142488 T500 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 142489 T500 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 142490 T500 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 142490 T500 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 142492 T500 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 142492 T500 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003/collection1/'
   [junit4]   2> 142494 T500 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 142495 T500 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 142496 T500 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest%20192E0B88CEF9FF07-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 142540 T500 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 142543 T500 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 142560 T500 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 142588 T500 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 142648 T500 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 142649 T500 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 142668 T500 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 143147 T500 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 143166 T500 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 143169 T500 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 143195 T500 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143202 T500 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143209 T500 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143210 T500 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143211 T500 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143212 T500 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143213 T500 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143213 T500 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143214 T500 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 143214 T500 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 143216 T500 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 192E0B88CEF9FF07-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenk

[...truncated too long message...]

eStreaming, LeaderInitiatedRecoveryOnCommitTest, TestLazyCores, TestCloudSchemaless, QueryResultKeyTest, TestDefaultStatsCache, PluginInfoTest, TestRawResponseWriter, QueryEqualityTest, TestRandomDVFaceting, PeerSyncTest, TestExtendedDismaxParser, BasicDistributedZk2Test, HardAutoCommitTest, TestDefaultSimilarityFactory, DocumentBuilderTest, SolrRequestParserTest, TestDynamicLoading, SolrIndexSplitterTest, TestNoOpRegenerator, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, LeaderElectionTest, ZkSolrClientTest, TestZkChroot, TestDistributedSearch, TestReload, TestRangeQuery, TestCoreContainer, SolrCmdDistributorTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFunctionQuery, DirectUpdateHandlerTest, TestIndexSearcher, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, AnalysisAfterCoreReloadTest, TestFoldingMultitermQuery, SpatialFilterTest, SolrCoreCheckLockOnStartupTest, DistanceFunctionTest, DebugComponentTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestQueryUtils, DirectSolrSpellCheckerTest, FileBasedSpellCheckerTest, TermsComponentTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, IndexSchemaTest, JSONWriterTest, TestAnalyzedSuggestions, CSVRequestHandlerTest, TestComponentsName, TestBinaryResponseWriter, SOLR749Test, AlternateDirectoryTest, TestQuerySenderNoQuery, SolrIndexConfigTest, MultiTermTest, TestBinaryField, MinimalSchemaTest, OutputWriterTest, NotRequiredUniqueKeyTest, TestCharFilters, EchoParamsTest, ScriptEngineTest, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.seed=192E0B88CEF9FF07 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ms -Dtests.timezone=Asia/Chungking -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | ChaosMonkeyNothingIsSafeTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:442)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:188)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   [junit4]    >    1) Thread[id=5844, name=qtp1510795348-5844, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:503)
   [junit4]    >         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
   [junit4]    >         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:228)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:773)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1770)
   [junit4]    >         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
   [junit4]    >         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2006)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:781)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:417)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:204)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    >         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    >         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=5854, name=searcherExecutor-2407-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([192E0B88CEF9FF07]:0)
   [junit4] Completed on J2 in 421.14s, 1 test, 1 failure, 1 error <<< FAILURES!

[...truncated 326 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 462 suites, 1871 tests, 2 suite-level errors, 5 errors, 57 ignored (9 assumptions)

Total time: 182 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 669999665 bytes
Compression is 0.0%
Took 2 min 3 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 733 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/733/

7 tests failed.
FAILED:  org.apache.solr.cloud.SaslZkACLProviderTest.testSaslZkACLProvider

Error Message:
KeeperErrorCode = ConnectionLoss for /solr

Stack Trace:
org.apache.zookeeper.KeeperException$ConnectionLossException: KeeperErrorCode = ConnectionLoss for /solr
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2:A091B71E392847EC]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:99)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:293)
	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:290)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:290)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:485)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:402)
	at org.apache.solr.cloud.SaslZkACLProviderTest.setUp(SaslZkACLProviderTest.java:80)
	at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:861)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2]:0)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2:25836FDED73384DE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2:25836FDED73384DE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:38670/zo/a/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38670/zo/a/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2:25836FDED73384DE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2:25836FDED73384DE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:365)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:320)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10241 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/init-core-data-001
   [junit4]   2> 763355 T1084 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zo/a
   [junit4]   2> 763362 T1084 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 763363 T1084 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 763364 T1085 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 763464 T1084 oasc.ZkTestServer.run start zk server on port:57641
   [junit4]   2> 763465 T1084 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 763466 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 763470 T1092 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c7ba954 name:ZooKeeperConnection Watcher:127.0.0.1:57641 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 763471 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 763471 T1084 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 763472 T1084 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 763475 T1084 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 763476 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 763478 T1095 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5df6abec name:ZooKeeperConnection Watcher:127.0.0.1:57641/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 763478 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 763479 T1084 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 763479 T1084 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 763481 T1084 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 763483 T1084 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 763484 T1084 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 763486 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 763487 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 763490 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 763491 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 763593 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 763594 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 763596 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 763597 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 763599 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 763599 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 763601 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 763602 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 763604 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 763604 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 763606 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 763607 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 763609 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 763610 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 763611 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 763612 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 763614 T1084 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 763615 T1084 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 764890 T1084 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 764895 T1084 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54181
   [junit4]   2> 764896 T1084 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 764896 T1084 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 764896 T1084 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002
   [junit4]   2> 764897 T1084 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002/'
   [junit4]   2> 764926 T1084 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002/solr.xml
   [junit4]   2> 764979 T1084 oasc.CoreContainer.<init> New CoreContainer 33344017
   [junit4]   2> 764980 T1084 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002/]
   [junit4]   2> 764981 T1084 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 764981 T1084 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 764981 T1084 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 764981 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 764981 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 764982 T1084 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 764982 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 764982 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 764982 T1084 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 764983 T1084 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 764983 T1084 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 764983 T1084 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 764984 T1084 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 764984 T1084 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 764984 T1084 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57641/solr
   [junit4]   2> 764985 T1084 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 764985 T1084 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 764986 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 764987 T1107 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b8f59a9 name:ZooKeeperConnection Watcher:127.0.0.1:57641 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 764988 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 764989 T1084 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 764991 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 764992 T1110 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cfab596 name:ZooKeeperConnection Watcher:127.0.0.1:57641/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 764993 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 764995 T1084 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 764997 T1084 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 764998 T1084 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 765000 T1084 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 765002 T1084 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 765004 T1084 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 765005 T1084 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30255_zo%2Fa
   [junit4]   2> 765006 T1084 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30255_zo%2Fa
   [junit4]   2> 765008 T1084 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 765009 T1084 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 765010 T1084 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 765012 T1084 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:30255_zo%2Fa
   [junit4]   2> 765012 T1084 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 765014 T1084 oasc.Overseer.start Overseer (id=93138280593096707-127.0.0.1:30255_zo%2Fa-n_0000000000) starting
   [junit4]   2> 765015 T1084 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 765021 T1084 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 765022 T1112 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 765022 T1084 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 765024 T1084 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 765025 T1084 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 765027 T1111 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 765032 T1114 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 765032 T1114 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 765033 T1114 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 765033 T1110 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 765035 T1111 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30255_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30255/zo/a",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 765036 T1111 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30255_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30255/zo/a",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 765036 T1111 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 765036 T1111 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 765037 T1110 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766033 T1114 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 766034 T1114 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 766034 T1114 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 766035 T1114 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 766035 T1114 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 766036 T1114 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002/collection1/'
   [junit4]   2> 766037 T1114 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 766038 T1114 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 766039 T1114 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 766075 T1114 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 766078 T1114 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 766092 T1114 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 766114 T1114 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 766162 T1114 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 766263 T1114 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 766277 T1114 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 766682 T1114 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 766698 T1114 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 766701 T1114 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 766714 T1114 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 766719 T1114 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 766723 T1114 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 766724 T1114 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 766725 T1114 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 766725 T1114 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 766726 T1114 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 766726 T1114 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 766727 T1114 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 766727 T1114 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 766727 T1114 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/control/data/
   [junit4]   2> 766728 T1114 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@61c6408d
   [junit4]   2> 766728 T1114 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/control/data
   [junit4]   2> 766729 T1114 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/control/data/index/
   [junit4]   2> 766729 T1114 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 766729 T1114 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/control/data/index
   [junit4]   2> 766730 T1114 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7477c8b4
   [junit4]   2> 766731 T1114 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6419dd28 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e06a06f),segFN=segments_1,generation=1}
   [junit4]   2> 766731 T1114 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 766736 T1114 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 766737 T1114 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 766737 T1114 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 766737 T1114 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 766737 T1114 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 766738 T1114 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 766738 T1114 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 766739 T1114 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 766739 T1114 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 766740 T1114 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 766740 T1114 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 766740 T1114 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 766741 T1114 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 766741 T1114 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 766742 T1114 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 766743 T1114 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 766743 T1114 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 766744 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 766745 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 766745 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 766746 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 766746 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 766747 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 766747 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 766748 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 766748 T1114 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 766749 T1114 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 766771 T1114 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766773 T1114 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766775 T1114 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766777 T1114 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766779 T1114 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 766784 T1114 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 766784 T1114 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 766785 T1114 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=1.1328125, floorSegmentMB=1.1328125, forceMergeDeletesPctAllowed=8.132005157091351, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30086876044175437
   [junit4]   2> 766786 T1114 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6419dd28 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5e06a06f),segFN=segments_1,generation=1}
   [junit4]   2> 766786 T1114 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 766786 T1114 oass.SolrIndexSearcher.<init> Opening Searcher@4fcd3f1a[collection1] main
   [junit4]   2> 766787 T1114 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 766787 T1114 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 766788 T1114 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 766788 T1114 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 766788 T1114 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 766789 T1114 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 766789 T1114 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 766789 T1114 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 766790 T1114 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 766790 T1114 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 766791 T1115 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4fcd3f1a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766792 T1114 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 766792 T1118 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30255/zo/a collection:control_collection shard:shard1
   [junit4]   2> 766793 T1084 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 766794 T1084 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 766794 T1118 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 766798 T1118 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 766798 T1084 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 766799 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 766800 T1110 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766801 T1118 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 766801 T1118 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 766801 T1122 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23c4775b name:ZooKeeperConnection Watcher:127.0.0.1:57641/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> ASYNC  NEW_CORE C439 name=collection1 org.apache.solr.core.SolrCore@3fb033fe url=http://127.0.0.1:30255/zo/a/collection1 node=127.0.0.1:30255_zo%2Fa C439_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:30255_zo%2Fa, base_url=http://127.0.0.1:30255/zo/a, state=down}
   [junit4]   2> 766801 T1118 C439 P30255 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30255/zo/a/collection1/
   [junit4]   2> 766801 T1111 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 766802 T1118 C439 P30255 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 766801 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 766802 T1118 C439 P30255 oasc.SyncStrategy.syncToMe http://127.0.0.1:30255/zo/a/collection1/ has no replicas
   [junit4]   2> 766802 T1118 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30255/zo/a/collection1/ shard1
   [junit4]   2> 766803 T1118 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 766803 T1084 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 766803 T1110 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766804 T1084 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 766807 T1123 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766807 T1084 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 766808 T1111 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30255/zo/a",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 766911 T1122 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766959 T1118 oasc.ZkController.register We are http://127.0.0.1:30255/zo/a/collection1/ and leader is http://127.0.0.1:30255/zo/a/collection1/
   [junit4]   2> 766960 T1118 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30255/zo/a
   [junit4]   2> 766960 T1118 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 766960 T1118 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 766960 T1118 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 766961 T1110 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766963 T1111 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30255_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30255/zo/a",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 766964 T1111 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30255_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30255/zo/a",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 767067 T1110 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 767067 T1122 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 768242 T1084 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 768243 T1084 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 768246 T1084 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:29964
   [junit4]   2> 768247 T1084 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 768247 T1084 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 768247 T1084 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003
   [junit4]   2> 768247 T1084 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003/'
   [junit4]   2> 768283 T1084 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003/solr.xml
   [junit4]   2> 768345 T1084 oasc.CoreContainer.<init> New CoreContainer 996050254
   [junit4]   2> 768346 T1084 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003/]
   [junit4]   2> 768346 T1084 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 768347 T1084 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 768347 T1084 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 768347 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 768347 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 768348 T1084 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 768348 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 768348 T1084 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 768348 T1084 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 768348 T1084 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 768349 T1084 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 768349 T1084 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 768349 T1084 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 768350 T1084 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 768350 T1084 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57641/solr
   [junit4]   2> 768350 T1084 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 768350 T1084 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 768351 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 768353 T1135 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58be205d name:ZooKeeperConnection Watcher:127.0.0.1:57641 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 768354 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 768354 T1084 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 768357 T1084 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 768358 T1138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@59778a6f name:ZooKeeperConnection Watcher:127.0.0.1:57641/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 768359 T1084 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 768364 T1084 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 769367 T1084 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43166_zo%2Fa
   [junit4]   2> 769368 T1084 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43166_zo%2Fa
   [junit4]   2> 769371 T1084 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 769379 T1139 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 769379 T1139 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 769380 T1110 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 769380 T1139 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 769381 T1111 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:43166_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43166/zo/a",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 769382 T1111 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:43166_zo%2Fa",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43166/zo/a",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 769382 T1111 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 769382 T1111 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 769384 T1110 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 769384 T1138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 769384 T1122 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 770381 T1139 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 770382 T1139 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 770382 T1139 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 770383 T1139 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 770383 T1139 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 770384 T1139 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003/collection1/'
   [junit4]   2> 770385 T1139 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 770386 T1139 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 770387 T1139 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest%20A465E1C6A06CE4E2-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 770422 T1139 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 770425 T1139 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 770439 T1139 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 770460 T1139 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 770507 T1139 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 770608 T1139 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 770622 T1139 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 771024 T1139 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 771039 T1139 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 771042 T1139 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 771055 T1139 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 771060 T1139 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 771064 T1139 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 771065 T1139 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 771066 T1139 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 771066 T1139 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 771067 T1139 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 771067 T1139 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 771068 T1139 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 771068 T1139 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 771068 T1139 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/jetty1/
   [junit4]   2> 771069 T1139 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@61c6408d
   [junit4]   2> 771070 T1139 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/jetty1
   [junit4]   2> 771070 T1139 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/jetty1/index/
   [junit4]   2> 771070 T1139 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 771070 T1139 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A465E1C6A06CE4E2-001/tempDir-001/jetty1/index
   [junit4]   2> 771071 T1139 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@50ec3323
   [junit4]   2> 771072 T1139 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43116856 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bd90880),segFN=segments_1,generation=1}
   [junit4]   2> 771072 T1139 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 771079 T1139 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 771079 T1139 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 771080 T1139 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 771080 T1139 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 771080 T1139 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 771080 T1139 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 771081 T1139 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 771081 T1139 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 771081 T1139 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 771082 T1139 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 771083 T1139 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 771083 T1139 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 771084 T1139 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 771085 T1139 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 771085 T1139 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 771087 T1139 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 771087 T1139 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 771088 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 771090 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 771090 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 771091 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 771092 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 771092 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 771093 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 771093 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 771094 T1139 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 771095 T1139 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 771115 T1139 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 771117 T1139 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 771119 T1139 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 771121 T1139 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 771123 T1139 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 771126 T1139 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 771126 T1139 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 771128 T1139 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=1.1328125, floorSegmentMB=1.1328125, forceMergeDeletesPctAllowed=8.132005157091351, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30086876044175437
   [junit4]   2> 771128 T1139 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43116856 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bd90880),segFN=segments_1,generation=1}
   [junit4]   2> 771129 T1139 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 771129 T1139 oass.SolrIndexSearcher.<init> Opening Searcher@39c896f4[collection1] main
   [junit4]   2> 771129 T1139 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 771130 T1139 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 771130 T1139 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 771131 T1139 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 771131 T1139 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 771131 T1139 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 771132 T1139 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 771132 T1139 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 771132 T1139 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 771133 T1139 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 771134 T1140 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@39c896f4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 771135 T1139 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 771135 T1143 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43166/zo/a collection:collection1 shard:shard1
   [junit4]   2>

[...truncated too long message...]

olExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  74) Thread[id=1085, name=Thread-350, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.lang.Object.wait(Object.java:503)
   [junit4]   2> 	        at org.apache.solr.core.CloserThread.run(CoreContainer.java:937)
   [junit4]   2> 	  75) Thread[id=1033, name=TEST-ChaosMonkeySafeLeaderTest.testDistribSearch-seed#[A465E1C6A06CE4E2]-SendThread(127.0.0.1:11641), state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.nio.ch.IOUtil.drain(Native Method)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.updateSelectedKeys(KQueueSelectorImpl.java:167)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:108)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 	  76) Thread[id=1068, name=qtp557197675-1068 Acceptor0 SocketConnector@127.0.0.1:12857, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  77) Thread[id=947, name=Thread-278, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.lang.Object.wait(Object.java:503)
   [junit4]   2> 	        at org.apache.solr.core.CloserThread.run(CoreContainer.java:937)
   [junit4]   2> 	  78) Thread[id=1205, name=commitScheduler-491-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  79) Thread[id=1196, name=commitScheduler-410-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  80) Thread[id=1021, name=HashSessionScavenger-22, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 	  81) Thread[id=1064, name=coreZkRegister-447-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), rnd_b=PostingsFormat(name=Asserting), a_i=PostingsFormat(name=Asserting), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=PostingsFormat(name=Asserting)}, docValues:{}, sim=DefaultSimilarity, locale=es_EC, timezone=AET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=83,free=92636728,total=249561088
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressUserVersions, ZkStateWriterTest, HdfsBasicDistributedZkTest, JSONWriterTest, SimplePostToolTest, CoreContainerCoreInitFailuresTest, ZkControllerTest, UniqFieldsUpdateProcessorFactoryTest, TestChildDocTransformer, TestInitParams, TestBM25SimilarityFactory, TestBinaryResponseWriter, TestQuerySenderNoQuery, TestStressLucene, SchemaVersionSpecificBehaviorTest, LegacyHTMLStripCharFilterTest, ExternalFileFieldSortTest, TestDynamicLoading, TestCloudSchemaless, BasicZkTest, TestUniqueKeyFieldResource, VMParamsZkACLAndCredentialsProvidersTest, ChaosMonkeySafeLeaderTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=A465E1C6A06CE4E2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_EC -Dtests.timezone=AET -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | ChaosMonkeySafeLeaderTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A465E1C6A06CE4E2]:0)
   [junit4] Completed on J1 in 7224.91s, 1 test, 2 errors <<< FAILURES!

[...truncated 114 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 461 suites, 1875 tests, 1 suite-level error, 6 errors, 171 ignored (7 assumptions)

Total time: 459 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 680731042 bytes
Compression is 0.0%
Took 2 min 4 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 732 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/732/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
ERROR: SolrIndexSearcher opens=628 closes=625

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=628 closes=625
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:442)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:188)
	at sun.reflect.GeneratedMethodAccessor41.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=3833, name=searcherExecutor-1911-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3830, name=searcherExecutor-1909-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=3835, name=searcherExecutor-1912-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
   1) Thread[id=3833, name=searcherExecutor-1911-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3830, name=searcherExecutor-1909-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=3835, name=searcherExecutor-1912-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=3833, name=searcherExecutor-1911-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3830, name=searcherExecutor-1909-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=3835, name=searcherExecutor-1912-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=3833, name=searcherExecutor-1911-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3830, name=searcherExecutor-1909-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=3835, name=searcherExecutor-1912-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:22008/qv_suz/f/collection1: parsing error

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:22008/qv_suz/f/collection1: parsing error
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69:47F2C186F1CB7D55]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:536)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:302)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:186)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:413)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:143)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: parsing error
	at org.apache.solr.client.solrj.impl.BinaryResponseParser.processResponse(BinaryResponseParser.java:45)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:534)
	... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.io.ChunkedInputStream.getChunkSize(ChunkedInputStream.java:228)
	at org.apache.http.impl.io.ChunkedInputStream.nextChunk(ChunkedInputStream.java:206)
	at org.apache.http.impl.io.ChunkedInputStream.read(ChunkedInputStream.java:169)
	at org.apache.http.conn.EofSensorInputStream.read(EofSensorInputStream.java:137)
	at org.apache.solr.common.util.FastInputStream.readWrappedStream(FastInputStream.java:80)
	at org.apache.solr.common.util.FastInputStream.refill(FastInputStream.java:89)
	at org.apache.solr.common.util.FastInputStream.read(FastInputStream.java:125)
	at org.apache.solr.common.util.FastInputStream.readFully(FastInputStream.java:152)
	at org.apache.solr.common.util.JavaBinCodec.readStr(JavaBinCodec.java:595)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:180)
	at org.apache.solr.common.util.JavaBinCodec.readSolrDocument(JavaBinCodec.java:360)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:219)
	at org.apache.solr.common.util.JavaBinCodec.readArray(JavaBinCodec.java:492)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:186)
	at org.apache.solr.common.util.JavaBinCodec.readSolrDocumentList(JavaBinCodec.java:374)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:221)
	at org.apache.solr.common.util.JavaBinCodec.readOrderedMap(JavaBinCodec.java:125)
	at org.apache.solr.common.util.JavaBinCodec.readVal(JavaBinCodec.java:188)
	at org.apache.solr.common.util.JavaBinCodec.unmarshal(JavaBinCodec.java:116)
	at org.apache.solr.client.solrj.impl.BinaryResponseParser.processResponse(BinaryResponseParser.java:43)
	... 50 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69:47F2C186F1CB7D55]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69:47F2C186F1CB7D55]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9663 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/init-core-data-001
   [junit4]   2> 52430 T58 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /qv_suz/f
   [junit4]   2> 52435 T58 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 52467 T58 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 52475 T59 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 52778 T58 oasc.ZkTestServer.run start zk server on port:35351
   [junit4]   2> 52921 T58 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 53208 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 53432 T66 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6dfee794 name:ZooKeeperConnection Watcher:127.0.0.1:35351 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53432 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 53433 T58 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 53450 T58 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 53648 T58 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 53650 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 53652 T69 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d99153f name:ZooKeeperConnection Watcher:127.0.0.1:35351/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53653 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 53653 T58 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 53676 T58 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 53681 T58 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 53684 T58 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 53687 T58 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 53692 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 53693 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 53699 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 53700 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 53705 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53705 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53712 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 53713 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 53717 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 53718 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 53721 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 53722 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 53726 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 53727 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 53731 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53732 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53735 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53736 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53740 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 53741 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 53745 T58 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 53745 T58 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 56011 T58 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 56022 T58 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22008
   [junit4]   2> 56023 T58 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 56023 T58 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 56024 T58 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002
   [junit4]   2> 56024 T58 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002/'
   [junit4]   2> 56084 T58 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002/solr.xml
   [junit4]   2> 56208 T58 oasc.CoreContainer.<init> New CoreContainer 339321761
   [junit4]   2> 56209 T58 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002/]
   [junit4]   2> 56211 T58 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 56211 T58 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 56211 T58 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 56212 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 56212 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 56213 T58 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 56213 T58 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 56214 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 56214 T58 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 56215 T58 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 56216 T58 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 56216 T58 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 56217 T58 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 56217 T58 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 56218 T58 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35351/solr
   [junit4]   2> 56293 T58 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 56293 T58 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 56295 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 56299 T81 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@705a2083 name:ZooKeeperConnection Watcher:127.0.0.1:35351 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 56299 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 56300 T58 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 56325 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 56327 T84 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6dbee707 name:ZooKeeperConnection Watcher:127.0.0.1:35351/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 56328 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 56355 T58 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 56376 T58 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 56389 T58 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 56393 T58 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 56398 T58 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 56428 T58 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 56431 T58 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22008_qv_suz%2Ff
   [junit4]   2> 56442 T58 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22008_qv_suz%2Ff
   [junit4]   2> 56447 T58 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 56450 T58 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 56454 T58 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 56483 T58 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22008_qv_suz%2Ff
   [junit4]   2> 56484 T58 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 56487 T58 oasc.Overseer.start Overseer (id=93131714592505859-127.0.0.1:22008_qv_suz%2Ff-n_0000000000) starting
   [junit4]   2> 56516 T58 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 56558 T58 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 56588 T86 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 56588 T58 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 56592 T58 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 56595 T58 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 56630 T85 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 56663 T88 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 56664 T88 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 56681 T84 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 56681 T88 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 56694 T85 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22008/qv_suz/f"} current state version: 0
   [junit4]   2> 56771 T85 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22008/qv_suz/f"}
   [junit4]   2> 56781 T85 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 56847 T85 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 56870 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 57683 T88 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 57683 T88 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 57684 T88 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 57685 T88 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 57686 T88 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 57687 T88 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002/collection1/'
   [junit4]   2> 57690 T88 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 57691 T88 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 57692 T88 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 57783 T88 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 57815 T88 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 57864 T88 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 57901 T88 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 57965 T88 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57967 T88 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 57988 T88 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 58448 T88 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 58452 T88 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 58455 T88 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58462 T88 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58528 T88 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 58529 T88 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 58530 T88 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/control/data/
   [junit4]   2> 58530 T88 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@66de9d91
   [junit4]   2> 58551 T88 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/control/data
   [junit4]   2> 58552 T88 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/control/data/index/
   [junit4]   2> 58552 T88 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 58553 T88 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/control/data/index
   [junit4]   2> 58553 T88 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=644136122, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 58555 T88 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1fa9bd6a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40d28a63),segFN=segments_1,generation=1}
   [junit4]   2> 58555 T88 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58566 T88 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 58738 T88 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 58739 T88 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 58739 T88 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58740 T88 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58740 T88 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 58741 T88 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58741 T88 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58742 T88 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 58754 T88 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58755 T88 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58756 T88 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58758 T88 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58759 T88 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 58760 T88 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 58761 T88 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 58763 T88 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 58764 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 58766 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 58768 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 58769 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 58771 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 58773 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 58774 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 58776 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 58778 T88 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 58780 T88 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 58810 T88 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58813 T88 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58815 T88 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58818 T88 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58820 T88 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 58826 T88 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 58826 T88 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 58828 T88 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1465173535, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 58829 T88 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1fa9bd6a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40d28a63),segFN=segments_1,generation=1}
   [junit4]   2> 58829 T88 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58830 T88 oass.SolrIndexSearcher.<init> Opening Searcher@3a90e7b[collection1] main
   [junit4]   2> 58830 T88 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 58832 T88 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 58832 T88 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 58850 T88 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 58850 T88 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 58851 T88 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58852 T88 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 58853 T88 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58853 T88 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 58853 T88 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 58856 T89 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3a90e7b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 58865 T88 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 58866 T92 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22008/qv_suz/f collection:control_collection shard:shard1
   [junit4]   2> 58867 T58 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 58868 T58 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 58913 T92 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 58924 T92 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 58928 T84 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58929 T92 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 58929 T92 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@1729443e url=http://127.0.0.1:22008/qv_suz/f/collection1 node=127.0.0.1:22008_qv_suz%2Ff C10_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:22008_qv_suz%2Ff, core=collection1, state=down, base_url=http://127.0.0.1:22008/qv_suz/f}
   [junit4]   2> 58930 T92 C10 P22008 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22008/qv_suz/f/collection1/
   [junit4]   2> 58930 T85 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 58931 T92 C10 P22008 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 58931 T92 C10 P22008 oasc.SyncStrategy.syncToMe http://127.0.0.1:22008/qv_suz/f/collection1/ has no replicas
   [junit4]   2> 58932 T92 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22008/qv_suz/f/collection1/ shard1
   [junit4]   2> 58941 T58 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 58942 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 58945 T95 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bf7d48f name:ZooKeeperConnection Watcher:127.0.0.1:35351/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 58946 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 58947 T58 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 58949 T58 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 58951 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58956 T92 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 58960 T58 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 58965 T96 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58967 T85 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22008/qv_suz/f",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 59075 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 59119 T92 oasc.ZkController.register We are http://127.0.0.1:22008/qv_suz/f/collection1/ and leader is http://127.0.0.1:22008/qv_suz/f/collection1/
   [junit4]   2> 59120 T92 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22008/qv_suz/f
   [junit4]   2> 59120 T92 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 59121 T92 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 59121 T92 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 59123 T84 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 59128 T85 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22008/qv_suz/f"} current state version: 3
   [junit4]   2> 59129 T85 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22008/qv_suz/f"}
   [junit4]   2> 59236 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 59236 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 61011 T58 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 61012 T58 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 61014 T58 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17008
   [junit4]   2> 61015 T58 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 61015 T58 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 61016 T58 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003
   [junit4]   2> 61016 T58 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003/'
   [junit4]   2> 61059 T58 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003/solr.xml
   [junit4]   2> 61142 T58 oasc.CoreContainer.<init> New CoreContainer 1515735772
   [junit4]   2> 61142 T58 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003/]
   [junit4]   2> 61143 T58 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 61143 T58 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 61144 T58 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 61144 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 61144 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 61145 T58 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 61145 T58 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 61145 T58 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 61146 T58 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 61146 T58 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 61147 T58 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 61147 T58 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 61147 T58 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 61148 T58 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 61148 T58 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35351/solr
   [junit4]   2> 61148 T58 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 61148 T58 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 61149 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61152 T108 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e6cd229 name:ZooKeeperConnection Watcher:127.0.0.1:35351 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61153 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61153 T58 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 61156 T58 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61158 T111 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7eac4f56 name:ZooKeeperConnection Watcher:127.0.0.1:35351/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61158 T58 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61168 T58 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 62174 T58 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17008_qv_suz%2Ff
   [junit4]   2> 62176 T58 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17008_qv_suz%2Ff
   [junit4]   2> 62182 T58 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 62200 T112 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 62200 T112 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 62202 T84 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 62202 T112 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 62204 T85 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:17008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17008/qv_suz/f"} current state version: 4
   [junit4]   2> 62205 T85 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:17008_qv_suz%2Ff",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17008/qv_suz/f"}
   [junit4]   2> 62205 T85 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 62206 T85 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 62209 T84 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 62209 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 62209 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 63203 T112 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 63204 T112 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 63205 T112 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 63206 T112 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 63207 T112 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 63207 T112 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003/collection1/'
   [junit4]   2> 63210 T112 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 63211 T112 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 63212 T112 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest%20C6144F9E86941D69-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 63264 T112 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 63269 T112 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 63304 T112 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 63334 T112 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 63398 T112 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 63401 T112 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 63422 T112 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 63884 T112 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 63888 T112 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 63891 T112 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63898 T112 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63962 T112 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 63963 T112 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 63963 T112 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/jetty1/
   [junit4]   2> 63963 T112 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@66de9d91
   [junit4]   2> 63965 T112 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/jetty1
   [junit4]   2> 63965 T112 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/jetty1/index/
   [junit4]   2> 63966 T112 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 63966 T112 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.FullSolrCloudDistribCmdsTest C6144F9E86941D69-001/tempDir-001/jetty1/index
   [junit4]   2> 63967 T112 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=644136122, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 63968 T112 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b52ec6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4511ca8d),segFN=segments_1,generation=1}
   [junit4]   2> 63968 T112 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 63981 T112 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 63982 T112 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 63983 T112 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 63983 T112 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63983 T112 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63984 T112 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 63984 T112 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63985 T112 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63985 T112 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 63987 T112 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63989 T112 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63991 T112 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63993 T112 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 63995 T112 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 63997 T112 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 64019 T112 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 64021 T112 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 64023 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 64026 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 64028 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 64030 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 64032 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 64034 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 64036 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 64038 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 64041 T112 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 64044 T112 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 64079 T112 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64082 T112 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64085 T112 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64088 T112 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64092 T112 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 64097 T112 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 64097 T112 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 64101 T112 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1465173535, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 64102 T112 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2b52ec6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4511ca8d),segFN=segments_1,generation=1}
   [junit4]   2> 64102 T112 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 64103 T112 oass.SolrIndexSearcher.<init> Opening Searcher@198b5597[collection1] main
   [junit4]   2> 64104 T112 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 64106 T112 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 64106 T112 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 64107 T112 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 64108 T112 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 64108 T112 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 64110 T112 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 64110 T112 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 64111 T112 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 64111 T112 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 64116 T113 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@198b5597[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 64117 T112 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 64118 T116 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17008/qv_suz/f collection:collection1 shard:shard2
   [junit4]   2> 64119 T58 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 64119 T58 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 64120 T116 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 64132 T116 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 64136 T84 oasc.DistributedQueue$LatchWatcher.process NodeChildr

[...truncated too long message...]

)
   [junit4]   2> 	        at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	        at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	        at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  49) Thread[id=2686, name=qtp1044064175-2686 Acceptor0 SocketConnector@127.0.0.1:14179, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  50) Thread[id=1809, name=HashSessionScavenger-31, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 	  51) Thread[id=2228, name=qtp1165881115-2228 Acceptor0 SocketConnector@127.0.0.1:48086, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  52) Thread[id=1996, name=commitScheduler-615-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1079)
   [junit4]   2> 	        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:807)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  53) Thread[id=2678, name=searcherExecutor-840-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=Direct), id=FSTOrd50, _version_=PostingsFormat(name=Memory doPackFST= false), a_i=FSTOrd50, rnd_b=FSTOrd50}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=sr_BA_#Latn, timezone=America/Menominee
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=55,free=172787512,total=419954688
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsUnloadDistributedZkTest, BJQParserTest, SimplePostToolTest, DistributedDebugComponentTest, TestSolrXml, TestFunctionQuery, PathHierarchyTokenizerFactoryTest, TestBinaryResponseWriter, FullSolrCloudDistribCmdsTest, TestRandomMergePolicy, TestSchemaNameResource, OverseerStatusTest, MinimalSchemaTest, AutoCommitTest, HdfsBasicDistributedZk2Test, SuggesterTSTTest, BadComponentTest, CloudExitableDirectoryReaderTest, DateFieldTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestExpandComponent, BasicZkTest, TestPerFieldSimilarity, TestHighlightDedupGrouping, FieldMutatingUpdateProcessorTest, OpenCloseCoreStressTest, JSONWriterTest, DOMUtilTest, TestSortingResponseWriter, ChaosMonkeySafeLeaderTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=C6144F9E86941D69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Menominee -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | ChaosMonkeySafeLeaderTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C6144F9E86941D69]:0)
   [junit4] Completed on J1 in 7224.28s, 1 test, 2 errors <<< FAILURES!

[...truncated 101 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 460 suites, 1872 tests, 4 suite-level errors, 4 errors, 885 ignored (5 assumptions)

Total time: 256 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 539101980 bytes
Compression is 0.0%
Took 1 min 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 731 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/731/

6 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10101/_v/b/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10101/_v/b/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B:17F9DC5CED78247]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10101/_v/b/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30430/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30430/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B:17F9DC5CED78247]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:77)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30430/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:40303/ss_/bl/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40303/ss_/bl/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B:17F9DC5CED78247]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:141)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27113/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27113/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B:17F9DC5CED78247]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27113/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B]:0)




Build Log:
[...truncated 9487 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/init-core-data-001
   [junit4]   2> 54527 T74 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 54535 T74 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 54571 T74 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 54582 T75 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 54882 T74 oasc.ZkTestServer.run start zk server on port:25032
   [junit4]   2> 55031 T74 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55300 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55520 T82 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63c9e656 name:ZooKeeperConnection Watcher:127.0.0.1:25032 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55521 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55522 T74 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 55537 T74 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 55698 T74 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55700 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55703 T85 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ae22b2 name:ZooKeeperConnection Watcher:127.0.0.1:25032/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55703 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55704 T74 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 55714 T74 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 55719 T74 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 55724 T74 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 55727 T74 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 55732 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 55777 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 55784 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 55785 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 55790 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 55791 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 55796 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 55797 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 55801 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 55802 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 55807 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 55808 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 55813 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 55814 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 55818 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 55819 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 55824 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 55825 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 55829 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 55830 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 55835 T74 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 55836 T74 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 59072 T74 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 59520 T74 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56791
   [junit4]   2> 59739 T74 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 59740 T74 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 59740 T74 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002
   [junit4]   2> 59740 T74 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002/'
   [junit4]   2> 59793 T74 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002/solr.xml
   [junit4]   2> 59883 T74 oasc.CoreContainer.<init> New CoreContainer 1591409408
   [junit4]   2> 59884 T74 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002/]
   [junit4]   2> 59994 T74 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 59994 T74 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 59995 T74 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 59995 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 59995 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 59996 T74 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 59996 T74 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 59996 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 59997 T74 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 59997 T74 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 59998 T74 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 59998 T74 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 59998 T74 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 59999 T74 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 59999 T74 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25032/solr
   [junit4]   2> 60065 T74 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 60065 T74 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 60066 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60070 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b47c4eb name:ZooKeeperConnection Watcher:127.0.0.1:25032 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60070 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60071 T74 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 60100 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60102 T101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25fbc3fb name:ZooKeeperConnection Watcher:127.0.0.1:25032/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60102 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60125 T74 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 60144 T74 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 60154 T74 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 60158 T74 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 60162 T74 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 60188 T74 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 60190 T74 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23558_
   [junit4]   2> 60200 T74 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23558_
   [junit4]   2> 60204 T74 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 60206 T74 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 60209 T74 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 60235 T74 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23558_
   [junit4]   2> 60236 T74 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 60238 T74 oasc.Overseer.start Overseer (id=93126571476910083-127.0.0.1:23558_-n_0000000000) starting
   [junit4]   2> 60264 T74 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 60304 T74 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 60328 T103 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 60329 T74 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 60332 T74 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 60334 T74 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 60361 T102 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 60384 T105 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 60384 T105 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 60397 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 60397 T105 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 60407 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:23558_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23558"} current state version: 0
   [junit4]   2> 60463 T102 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:23558_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23558"}
   [junit4]   2> 60470 T102 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 60527 T102 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 60549 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 61399 T105 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 61399 T105 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 61400 T105 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 61400 T105 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 61402 T105 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 61402 T105 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002/collection1/'
   [junit4]   2> 61405 T105 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 61405 T105 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 61406 T105 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 61475 T105 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 61500 T105 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 61536 T105 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 61558 T105 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 61594 T105 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 61596 T105 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 61608 T105 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 61889 T105 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 61900 T105 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 61903 T105 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 61925 T105 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 61930 T105 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 61934 T105 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 61935 T105 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 61936 T105 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 61936 T105 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 61938 T105 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 61938 T105 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 61938 T105 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 61938 T105 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 61939 T105 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/control/data/
   [junit4]   2> 61939 T105 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ecc926e
   [junit4]   2> 61951 T105 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/control/data
   [junit4]   2> 61952 T105 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/control/data/index/
   [junit4]   2> 61953 T105 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 61953 T105 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/control/data/index
   [junit4]   2> 61953 T105 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7137641047317194]
   [junit4]   2> 61954 T105 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7c4e52e1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7a237188),segFN=segments_1,generation=1}
   [junit4]   2> 61955 T105 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 61962 T105 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 62041 T105 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 62041 T105 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 62042 T105 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 62042 T105 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 62042 T105 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 62043 T105 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 62043 T105 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 62043 T105 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 62050 T105 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 62050 T105 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 62051 T105 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 62052 T105 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 62053 T105 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 62054 T105 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 62055 T105 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 62055 T105 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 62056 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 62057 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 62058 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 62059 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 62060 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 62061 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 62062 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 62063 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 62063 T105 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 62064 T105 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 62079 T105 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 62080 T105 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 62082 T105 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 62084 T105 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 62085 T105 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 62088 T105 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 62088 T105 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 62090 T105 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=41.3388671875, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=1.4786152477673564, segmentsPerTier=30.0, maxCFSSegmentSizeMB=1.9638671875, noCFSRatio=0.5576594075430266
   [junit4]   2> 62091 T105 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7c4e52e1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7a237188),segFN=segments_1,generation=1}
   [junit4]   2> 62091 T105 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 62091 T105 oass.SolrIndexSearcher.<init> Opening Searcher@f997fe2[collection1] main
   [junit4]   2> 62092 T105 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 62093 T105 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 62094 T105 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 62103 T105 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 62103 T105 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 62103 T105 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 62104 T105 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 62104 T105 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 62105 T105 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 62105 T105 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 62108 T106 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@f997fe2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 62114 T105 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 62115 T109 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23558 collection:control_collection shard:shard1
   [junit4]   2> 62115 T74 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 62116 T74 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 62163 T109 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 62172 T109 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 62175 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 62176 T109 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 62176 T109 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 62176 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@16cf2484 url=http://127.0.0.1:23558/collection1 node=127.0.0.1:23558_ C4_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:23558_, state=down, base_url=http://127.0.0.1:23558}
   [junit4]   2> 62187 T109 C4 P23558 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23558/collection1/
   [junit4]   2> 62188 T109 C4 P23558 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 62188 T109 C4 P23558 oasc.SyncStrategy.syncToMe http://127.0.0.1:23558/collection1/ has no replicas
   [junit4]   2> 62189 T109 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23558/collection1/ shard1
   [junit4]   2> 62190 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 62206 T109 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 62213 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 62215 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23558",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 62267 T74 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 62268 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 62270 T114 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ec4b942 name:ZooKeeperConnection Watcher:127.0.0.1:25032/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 62270 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 62271 T74 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 62273 T74 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 62282 T74 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 62321 T114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 62367 T109 oasc.ZkController.register We are http://127.0.0.1:23558/collection1/ and leader is http://127.0.0.1:23558/collection1/
   [junit4]   2> 62368 T109 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23558
   [junit4]   2> 62368 T109 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 62368 T109 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 62369 T109 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 62371 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 62375 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:23558_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23558"} current state version: 3
   [junit4]   2> 62376 T102 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:23558_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23558"}
   [junit4]   2> 62482 T114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 62482 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 63818 T74 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 63820 T74 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 63823 T74 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62670
   [junit4]   2> 63824 T74 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 63824 T74 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 63825 T74 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003
   [junit4]   2> 63825 T74 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003/'
   [junit4]   2> 63890 T74 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003/solr.xml
   [junit4]   2> 64003 T74 oasc.CoreContainer.<init> New CoreContainer 2117768142
   [junit4]   2> 64003 T74 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003/]
   [junit4]   2> 64005 T74 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 64005 T74 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 64006 T74 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 64006 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 64007 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 64007 T74 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 64008 T74 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 64008 T74 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 64009 T74 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 64009 T74 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 64010 T74 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 64011 T74 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 64011 T74 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 64012 T74 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 64012 T74 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25032/solr
   [junit4]   2> 64013 T74 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 64013 T74 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 64015 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64017 T126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ca6c772 name:ZooKeeperConnection Watcher:127.0.0.1:25032 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64018 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64019 T74 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 64022 T74 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64024 T129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31420e20 name:ZooKeeperConnection Watcher:127.0.0.1:25032/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64025 T74 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64036 T74 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 65043 T74 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27113_
   [junit4]   2> 65045 T74 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27113_
   [junit4]   2> 65051 T74 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 65069 T130 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 65069 T130 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 65071 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 65071 T130 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 65073 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:27113_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27113"} current state version: 4
   [junit4]   2> 65074 T102 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:27113_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27113"}
   [junit4]   2> 65074 T102 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 65075 T102 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 65078 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 65078 T114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 65078 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66072 T130 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 66073 T130 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 66074 T130 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 66074 T130 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 66076 T130 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 66077 T130 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003/collection1/'
   [junit4]   2> 66079 T130 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 66080 T130 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 66081 T130 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%20809913DDB988E27B-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 66133 T130 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 66137 T130 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 66156 T130 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 66184 T130 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 66257 T130 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 66260 T130 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 66273 T130 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 66728 T130 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 66744 T130 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 66747 T130 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 66772 T130 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 66777 T130 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 66782 T130 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 66784 T130 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 66784 T130 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 66785 T130 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 66786 T130 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 66787 T130 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 66787 T130 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 66787 T130 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 66788 T130 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/jetty1/
   [junit4]   2> 66788 T130 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ecc926e
   [junit4]   2> 66791 T130 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/jetty1
   [junit4]   2> 66793 T130 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/jetty1/index/
   [junit4]   2> 66793 T130 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 66793 T130 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 809913DDB988E27B-001/tempDir-001/jetty1/index
   [junit4]   2> 66794 T130 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7137641047317194]
   [junit4]   2> 66795 T130 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41ce0390 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c4c04b0),segFN=segments_1,generation=1}
   [junit4]   2> 66796 T130 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 66810 T130 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 66810 T130 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 66811 T130 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 66811 T130 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 66811 T130 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 66812 T130 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 66812 T130 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 66813 T130 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 66813 T130 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 66815 T130 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66817 T130 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66819 T130 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66821 T130 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66823 T130 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 66825 T130 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 66839 T130 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 66841 T130 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 66843 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 66846 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 66848 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 66850 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 66852 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 66854 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 66856 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 66858 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 66859 T130 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 66861 T130 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 66882 T130 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66885 T130 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66887 T130 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66889 T130 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66891 T130 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 66894 T130 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 66894 T130 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 66897 T130 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=41.3388671875, floorSegmentMB=0.611328125, forceMergeDeletesPctAllowed=1.4786152477673564, segmentsPerTier=30.0, maxCFSSegmentSizeMB=1.9638671875, noCFSRatio=0.5576594075430266
   [junit4]   2> 66898 T130 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41ce0390 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c4c04b0),segFN=segments_1,generation=1}
   [junit4]   2> 66898 T130 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 66899 T130 oass.SolrIndexSearcher.<init> Opening Searcher@75bb42bf[collection1] main
   [junit4]   2> 66899 T130 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 66901 T130 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 66901 T130 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 66902 T130 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 66902 T130 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 66903 T130 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 66904 T130 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 66904 T130 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 66904 T130 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 66905 T130 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 66908 T131 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@75bb42bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 66908 T130 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 66909 T134 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27113 collection:collection1 shard:shard2
   [junit4]   2> 66910 T74 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 66910 T74 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 66911 T134 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 66919 T134 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 66921 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 66922 T134 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 66922 T134 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@7b49abe5 url=http://127.0.0.1:27113/collection1 node=127.0.0.1:27113_ C5_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:27113_, state=down, base_url=http://127.0.0.1:27113}
   [junit4]   2> 66922 T134 C5 P27113 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27113/collection1/
   [junit4]   2> 66923 T134 C5 P27113 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 66923 T134 C5 P27113 oasc.SyncStrategy.syncToMe http://127.0.0.1:27113/collection1/ has no replicas
   [junit4]   2> 66923 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 66923 T134 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27113/collection1/ shard2
   [junit4]   2> 66924 T134 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 66926 T114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66926 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66926 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66933 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27113",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 66937 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 66938 T129 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66938 T114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 66984 T134 oasc.ZkController.register We are http://127.0.0.1:27113/collection1/ and leader is http://127.0.0.1:27113/collection1/
   [junit4]   2> 66985 T134 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27113
   [junit4]   2> 66985 T134 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 66985 T134 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 66985 T134 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 66987 T101 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 66989 T102 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:27113_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27113"} current state version: 7
   [junit4]   2> 66990 T102 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core"

[...truncated too long message...]

TestRuleStoreClassName.java:42)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 8437808 T4384 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 8437809 T4384 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 8437809 T4384 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data;done=false>>]
   [junit4]   2> 8437810 T4384 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data
   [junit4]   2> 8437810 T4384 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data/index;done=false>>]
   [junit4]   2> 8437810 T4384 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001/solr-instance-001/collection1/data/index
   [junit4]   2> 8437811 T4384 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 8437830 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestIndexAndConfigAliasReplication
   [junit4]   2> 8437840 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestIndexAndConfigAliasReplication
   [junit4]   2> 8437847 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestSnapPullWithMasterUrl
   [junit4]   2> 8437851 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestSnapPullWithMasterUrl
   [junit4]   2> 8437857 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestDetails
   [junit4]   2> 8437865 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestDetails
   [junit4]   2> 8437873 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterWrite2Slave
   [junit4]   2> 8437881 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestReplicateAfterWrite2Slave
   [junit4]   2> 8437887 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterStartup
   [junit4]   2> 8437895 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestReplicateAfterStartup
   [junit4]   2> 8437901 T4384 oas.SolrTestCaseJ4.setUp ###Starting testEmptyCommits
   [junit4]   2> 8437909 T4384 oas.SolrTestCaseJ4.tearDown ###Ending testEmptyCommits
   [junit4]   2> 8437915 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestIndexAndConfigReplication
   [junit4]   2> 8437924 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestIndexAndConfigReplication
   [junit4]   2> 8437931 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterStartupWithNoActivity
   [junit4]   2> 8437939 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestReplicateAfterStartupWithNoActivity
   [junit4]   2> 8437945 T4384 oas.SolrTestCaseJ4.setUp ###Starting doTestStressReplication
   [junit4]   2> 8437954 T4384 oas.SolrTestCaseJ4.tearDown ###Ending doTestStressReplication
   [junit4]   2> 8437956 T4384 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 809913DDB988E27B-001
   [junit4]   2> 7221296 T4383 ccr.ThreadLeakControl.tryToInterruptAll All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {name=FST50, id=FST50}, docValues:{}, sim=DefaultSimilarity, locale=es_AR, timezone=NET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=332861864,total=467664896
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedQueryElevationComponentTest, TestAddFieldRealTimeGet, SignatureUpdateProcessorFactoryTest, TestFuzzyAnalyzedSuggestions, TestLRUStatsCache, RegexBoostProcessorTest, TestConfigReload, TestSolrConfigHandlerCloud, TestShardHandlerFactory, BasicDistributedZkTest, TestSort, CollectionsAPIAsyncDistributedZkTest, CopyFieldTest, StressHdfsTest, TestExceedMaxTermLength, TestLFUCache, MBeansHandlerTest, TestSuggestSpellingConverter, OverseerCollectionProcessorTest, TestManagedSchemaFieldTypeResource, XmlUpdateRequestHandlerTest, SpellingQueryConverterTest, DebugComponentTest, DeleteShardTest, CoreAdminCreateDiscoverTest, TestCollationField, TestCollapseQParserPlugin, TimeZoneUtilsTest, TestCollationFieldDocValues, TestCollectionAPI, HdfsUnloadDistributedZkTest, DirectSolrSpellCheckerTest, AliasIntegrationTest, TestExpandComponent, TestRecoveryHdfs, ZkStateWriterTest, TestMergePolicyConfig, ParsingFieldUpdateProcessorsTest, TestHighlightDedupGrouping, TestRandomFaceting, TestSchemaResource, TestStressReorder, TestClassNameShortening, TestAtomicUpdateErrorCases, ChangedSchemaMergeTest, StatelessScriptUpdateProcessorFactoryTest, TestRTGBase, TestIBSimilarityFactory, DistributedDebugComponentTest, TestFastLRUCache, DistributedSpellCheckComponentTest, DistributedFacetPivotLargeTest, AnalyticsMergeStrategyTest, RequestHandlersTest, ShardSplitTest, TestDFRSimilarityFactory, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, TestReplicationHandler]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReplicationHandler -Dtests.seed=809913DDB988E27B -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_AR -Dtests.timezone=NET -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestReplicationHandler (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([809913DDB988E27B]:0)
   [junit4] Completed on J0 in 7221.33s, 4 tests, 2 errors <<< FAILURES!

[...truncated 17 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 459 suites, 1861 tests, 1 suite-level error, 5 errors, 57 ignored (10 assumptions)

Total time: 242 minutes 15 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 632091247 bytes
Compression is 0.0%
Took 1 min 54 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 730 - Still Failing

Posted by Michael McCandless <lu...@mikemccandless.com>.
Thanks Rob.  This test does easily eat file descriptors...

Mike McCandless

http://blog.mikemccandless.com

On Sat, Jan 10, 2015 at 3:31 PM, Robert Muir <rc...@gmail.com> wrote:
> I committed a fix.
>
> On Sat, Jan 10, 2015 at 9:44 AM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/730/
>>
>> 1 tests failed.
>> REGRESSION:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField
>>
>> Error Message:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files
>>
>> Stack Trace:
>> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files
>>         at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>         at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>         at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>         at java.nio.file.Files.newOutputStream(Files.java:172)
>>         at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>         at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>         at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.<init>(BlockTreeTermsWriter.java:278)
>>         at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsConsumer(Lucene50PostingsFormat.java:433)
>>         at org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat.fieldsConsumer(BloomFilteringPostingsFormat.java:147)
>>         at org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings.fieldsConsumer(TestBloomFilteredLucenePostings.java:66)
>>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:196)
>>         at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107)
>>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:112)
>>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>         at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:373)
>>         at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:466)
>>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>         at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1076)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 1862 lines...]
>>    [junit4] Suite: org.apache.lucene.index.TestDemoParallelLeafReader
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #1,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #12,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #6,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #16,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #14,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #4,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #2,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #5,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #7,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #20,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #21,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #19,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #0,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #13,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #8,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #15,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #3,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #11,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #10,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #18,5,TGRP-TestDemoParallelLeafReader]
>>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2>
>>    [junit4]   2> jan 10, 2015 7:12:20 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>>    [junit4]   2> WARNING: Uncaught exception
>>
>> [...truncated too long message...]
>>
>> Wrapper.createOutput(MockDirectoryWrapper.java:607)
>>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>>    [junit4]    >        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), number=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{number_1=DocValuesFormat(name=Lucene50), number_4=DocValuesFormat(name=Asserting), number_53=DocValuesFormat(name=Asserting), number_84=DocValuesFormat(name=Lucene50), number_78=DocValuesFormat(name=Lucene50), number_83=DocValuesFormat(name=Memory), number_71=DocValuesFormat(name=Asserting), number_0=DocValuesFormat(name=Asserting), number_87=DocValuesFormat(name=Memory), number_30=DocValuesFormat(name=Lucene50), number_67=DocValuesFormat(name=Lucene50), number_57=DocValuesFormat(name=Asserting), number_20=DocValuesFormat(name=Asserting), number_69=DocValuesFormat(name=Memory), number_59=DocValuesFormat(name=Lucene50), number_73=DocValuesFormat(name=Lucene50), number_77=DocValuesFormat(name=Lucene50), number_175=DocValuesFormat(name=Memory), number_130=DocValuesFormat(name=Lucene50), number_49=DocValuesFormat(name=Lucene50), number_88=DocValuesFormat(name=Lucene50), number_70=DocValuesFormat(name=Lucene50), number_82=DocValuesFormat(name=Asserting), number_86=DocValuesFormat(name=Asserting), number_3=DocValuesFormat(name=Memory), number_92=DocValuesFormat(name=Lucene50), number_74=DocValuesFormat(name=Lucene50), number_198=DocValuesFormat(name=Asserting), number_160=DocValuesFormat(name=Memory), number_68=DocValuesFormat(name=Asserting), number_75=DocValuesFormat(name=Asserting), number_7=DocValuesFormat(name=Memory), number_76=DocValuesFormat(name=Memory), number_79=DocValuesFormat(name=Asserting), number_2=DocValuesFormat(name=Lucene50), number_72=DocValuesFormat(name=Memory), number_22=DocValuesFormat(name=Lucene50), number_99=DocValuesFormat(name=Lucene50), number_90=DocValuesFormat(name=Memory), number_66=DocValuesFormat(name=Lucene50), number_80=DocValuesFormat(name=Lucene50), number_91=DocValuesFormat(name=Lucene50), number=DocValuesFormat(name=Memory), number_81=DocValuesFormat(name=Lucene50), number_42=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=sk, timezone=Asia/Ashgabat
>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=138795984,total=459276288
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestTotalHitCountCollector, TestPhraseQuery, TestDirectoryReaderReopen, TestSimpleAttributeImpl, TestBasics, TestRegExp, TestSnapshotDeletionPolicy, TestBoolean2, Nested1, TestRoaringDocIdSet, Test2BSortedDocValues, TestPositiveScoresOnlyCollector, TestUnicodeUtil, TestPerFieldPostingsFormat2, TestWildcard, TestLucene50NormsFormat, TestBitDocIdSetBuilder, TestDocsAndPositions, Nested1, TestBagOfPostings, TestPayloadNearQuery, TestFilteredQuery, TestExternalCodecs, TestIndexWriterConfig, TestFieldCacheTermsFilter, TestWorstCaseTestBehavior, TestFlex, TestPersistentSnapshotDeletionPolicy, TestCachingWrapperFilter, Test2BTerms, TestMultiTermQueryRewrites, TestSpanExplanationsOfNonMatches, TestBinaryTerms, TestSpanSearchEquivalence, TestBufferedIndexInput, TestMultiLevelSkipList, TestSimilarityBase, TestDocInverterPerFieldErrorInfo, TestAssertingDocValuesFormat, TestStringHelper, TestBinaryDocument, TestDeterminizeLexicon, TestIndexWriterWithThreads, TestSpansAdvanced, TestWindowsMMap, TestSearch, Nested1, TestNeverDelete, TestDocIdSetBuilder, TestNoDeletionPolicy, TestIndexWriterOnJRECrash, TestIndexWriterReader, TestCompressingTermVectorsFormat, TestIndexWriterMerging, TestFSTs, TestPackedInts, TestMultiMMap, TestFlushByRamOrCountsPolicy, TestSloppyPhraseQuery, TestIndexWriterDelete, TestNorms, TestSearchWithThreads, TestSearchAfter, TestConcurrentMergeScheduler, TestHighCompressionMode, TestDocumentsWriterStallControl, TestTopDocsMerge, TestStressIndexing, TestMinimize, TestRollingUpdates, TestNRTReaderWithThreads, TestPerFieldPostingsFormat, TestConsistentFieldNumbers, TestSimpleExplanations, TestDocTermOrdsRangeFilter, TestCompressingStoredFieldsFormat, TestSegmentMerger, TestBagOfPositions, TestFixedBitSet, TestNRTCachingDirectory, TestIndexWriterNRTIsCurrent, TestPostingsOffsets, TestMixedCodecs, TestTransactionRollback, TestUTF32ToUTF8, TestSpanExplanations, TestStressIndexing2, TestMultiPhraseQuery, TestLazyProxSkipping, TestDeterminism, TestCodecs, TestCharTermAttributeImpl, TestOmitPositions, TestPayloadExplanations, TestParallelLeafReader, TestBytesRefHash, TestThreadedForceMerge, TestIndexInput, TestSloppyPhraseQuery2, TestDisjunctionMaxQuery, TestForceMergeForever, TestSameScoresWithThreads, TestQueryWrapperFilter, TestMultiFields, TestSimpleExplanationsOfNonMatches, TestTermVectorsReader, TestSpansAdvanced2, TestIntBlockPool, TestPrefixInBooleanQuery, TestTopDocsCollector, TestSegmentTermEnum, TestUniqueTermCount, TestBooleanQuery, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestDocument, TestTermScorer, TestTopFieldCollector, TestSumDocFreq, TestIndexableField, TestIndexFileDeleter, TestMatchAllDocsQuery, TestParallelTermEnum, TestSubScorerFreqs, Nested, Nested, TestDateSort, TestVirtualMethod, TestDateTools, TestAutomatonQueryUnicode, TestMockDirectoryWrapper, TestNoMergePolicy, TestNamedSPILoader, TestBytesRef, TestNot, TestVersion, TestScoreCachingWrappingScorer, TestTermdocPerf, TestIndexCommit, TestNewestSegment, Test2BBinaryDocValues, TestReusableStringReader, TestBlockPostingsFormat, TestForUtil, TestLucene50CompoundFormat, TestFieldType, TestAllFilesHaveChecksumFooter, TestCodecUtil, TestDemoParallelLeafReader]
>>    [junit4] Completed on J3 in 610.62s, 5 tests, 1 error <<< FAILURES!
>>
>> [...truncated 15 lines...]
>> BUILD FAILED
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 409 suites, 3237 tests, 1 error, 37 ignored (27 assumptions)
>>
>> Total time: 39 minutes 43 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>> Archived 5 artifacts
>> Archive block size is 32768
>> Received 0 blocks and 3856023 bytes
>> Compression is 0.0%
>> Took 1.5 sec
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>>
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 730 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I committed a fix.

On Sat, Jan 10, 2015 at 9:44 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/730/
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField
>
> Error Message:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files
>
> Stack Trace:
> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files
>         at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>         at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>         at java.nio.file.Files.newOutputStream(Files.java:172)
>         at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>         at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>         at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.<init>(BlockTreeTermsWriter.java:278)
>         at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsConsumer(Lucene50PostingsFormat.java:433)
>         at org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat.fieldsConsumer(BloomFilteringPostingsFormat.java:147)
>         at org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings.fieldsConsumer(TestBloomFilteredLucenePostings.java:66)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:196)
>         at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107)
>         at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:112)
>         at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>         at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>         at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:373)
>         at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:466)
>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>         at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1076)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 1862 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestDemoParallelLeafReader
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #1,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #12,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #6,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #16,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #14,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #4,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #2,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #5,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #7,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #20,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]   2>        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #21,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #19,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #0,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #13,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #8,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #15,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #3,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
>    [junit4]   2>        at java.nio.file.Files.newDirectoryStream(Files.java:545)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
>    [junit4]   2>        at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #11,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #10,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #18,5,TGRP-TestDemoParallelLeafReader]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
>    [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
>    [junit4]   2>        at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
>    [junit4]   2>        at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
>    [junit4]   2>        at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]   2>        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2>
>    [junit4]   2> jan 10, 2015 7:12:20 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception
>
> [...truncated too long message...]
>
> Wrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
>    [junit4]    >        at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
>    [junit4]    >        at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), number=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{number_1=DocValuesFormat(name=Lucene50), number_4=DocValuesFormat(name=Asserting), number_53=DocValuesFormat(name=Asserting), number_84=DocValuesFormat(name=Lucene50), number_78=DocValuesFormat(name=Lucene50), number_83=DocValuesFormat(name=Memory), number_71=DocValuesFormat(name=Asserting), number_0=DocValuesFormat(name=Asserting), number_87=DocValuesFormat(name=Memory), number_30=DocValuesFormat(name=Lucene50), number_67=DocValuesFormat(name=Lucene50), number_57=DocValuesFormat(name=Asserting), number_20=DocValuesFormat(name=Asserting), number_69=DocValuesFormat(name=Memory), number_59=DocValuesFormat(name=Lucene50), number_73=DocValuesFormat(name=Lucene50), number_77=DocValuesFormat(name=Lucene50), number_175=DocValuesFormat(name=Memory), number_130=DocValuesFormat(name=Lucene50), number_49=DocValuesFormat(name=Lucene50), number_88=DocValuesFormat(name=Lucene50), number_70=DocValuesFormat(name=Lucene50), number_82=DocValuesFormat(name=Asserting), number_86=DocValuesFormat(name=Asserting), number_3=DocValuesFormat(name=Memory), number_92=DocValuesFormat(name=Lucene50), number_74=DocValuesFormat(name=Lucene50), number_198=DocValuesFormat(name=Asserting), number_160=DocValuesFormat(name=Memory), number_68=DocValuesFormat(name=Asserting), number_75=DocValuesFormat(name=Asserting), number_7=DocValuesFormat(name=Memory), number_76=DocValuesFormat(name=Memory), number_79=DocValuesFormat(name=Asserting), number_2=DocValuesFormat(name=Lucene50), number_72=DocValuesFormat(name=Memory), number_22=DocValuesFormat(name=Lucene50), number_99=DocValuesFormat(name=Lucene50), number_90=DocValuesFormat(name=Memory), number_66=DocValuesFormat(name=Lucene50), number_80=DocValuesFormat(name=Lucene50), number_91=DocValuesFormat(name=Lucene50), number=DocValuesFormat(name=Memory), number_81=DocValuesFormat(name=Lucene50), number_42=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=sk, timezone=Asia/Ashgabat
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=138795984,total=459276288
>    [junit4]   2> NOTE: All tests run in this JVM: [TestTotalHitCountCollector, TestPhraseQuery, TestDirectoryReaderReopen, TestSimpleAttributeImpl, TestBasics, TestRegExp, TestSnapshotDeletionPolicy, TestBoolean2, Nested1, TestRoaringDocIdSet, Test2BSortedDocValues, TestPositiveScoresOnlyCollector, TestUnicodeUtil, TestPerFieldPostingsFormat2, TestWildcard, TestLucene50NormsFormat, TestBitDocIdSetBuilder, TestDocsAndPositions, Nested1, TestBagOfPostings, TestPayloadNearQuery, TestFilteredQuery, TestExternalCodecs, TestIndexWriterConfig, TestFieldCacheTermsFilter, TestWorstCaseTestBehavior, TestFlex, TestPersistentSnapshotDeletionPolicy, TestCachingWrapperFilter, Test2BTerms, TestMultiTermQueryRewrites, TestSpanExplanationsOfNonMatches, TestBinaryTerms, TestSpanSearchEquivalence, TestBufferedIndexInput, TestMultiLevelSkipList, TestSimilarityBase, TestDocInverterPerFieldErrorInfo, TestAssertingDocValuesFormat, TestStringHelper, TestBinaryDocument, TestDeterminizeLexicon, TestIndexWriterWithThreads, TestSpansAdvanced, TestWindowsMMap, TestSearch, Nested1, TestNeverDelete, TestDocIdSetBuilder, TestNoDeletionPolicy, TestIndexWriterOnJRECrash, TestIndexWriterReader, TestCompressingTermVectorsFormat, TestIndexWriterMerging, TestFSTs, TestPackedInts, TestMultiMMap, TestFlushByRamOrCountsPolicy, TestSloppyPhraseQuery, TestIndexWriterDelete, TestNorms, TestSearchWithThreads, TestSearchAfter, TestConcurrentMergeScheduler, TestHighCompressionMode, TestDocumentsWriterStallControl, TestTopDocsMerge, TestStressIndexing, TestMinimize, TestRollingUpdates, TestNRTReaderWithThreads, TestPerFieldPostingsFormat, TestConsistentFieldNumbers, TestSimpleExplanations, TestDocTermOrdsRangeFilter, TestCompressingStoredFieldsFormat, TestSegmentMerger, TestBagOfPositions, TestFixedBitSet, TestNRTCachingDirectory, TestIndexWriterNRTIsCurrent, TestPostingsOffsets, TestMixedCodecs, TestTransactionRollback, TestUTF32ToUTF8, TestSpanExplanations, TestStressIndexing2, TestMultiPhraseQuery, TestLazyProxSkipping, TestDeterminism, TestCodecs, TestCharTermAttributeImpl, TestOmitPositions, TestPayloadExplanations, TestParallelLeafReader, TestBytesRefHash, TestThreadedForceMerge, TestIndexInput, TestSloppyPhraseQuery2, TestDisjunctionMaxQuery, TestForceMergeForever, TestSameScoresWithThreads, TestQueryWrapperFilter, TestMultiFields, TestSimpleExplanationsOfNonMatches, TestTermVectorsReader, TestSpansAdvanced2, TestIntBlockPool, TestPrefixInBooleanQuery, TestTopDocsCollector, TestSegmentTermEnum, TestUniqueTermCount, TestBooleanQuery, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestDocument, TestTermScorer, TestTopFieldCollector, TestSumDocFreq, TestIndexableField, TestIndexFileDeleter, TestMatchAllDocsQuery, TestParallelTermEnum, TestSubScorerFreqs, Nested, Nested, TestDateSort, TestVirtualMethod, TestDateTools, TestAutomatonQueryUnicode, TestMockDirectoryWrapper, TestNoMergePolicy, TestNamedSPILoader, TestBytesRef, TestNot, TestVersion, TestScoreCachingWrappingScorer, TestTermdocPerf, TestIndexCommit, TestNewestSegment, Test2BBinaryDocValues, TestReusableStringReader, TestBlockPostingsFormat, TestForUtil, TestLucene50CompoundFormat, TestFieldType, TestAllFilesHaveChecksumFooter, TestCodecUtil, TestDemoParallelLeafReader]
>    [junit4] Completed on J3 in 610.62s, 5 tests, 1 error <<< FAILURES!
>
> [...truncated 15 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 409 suites, 3237 tests, 1 error, 37 ignored (27 assumptions)
>
> Total time: 39 minutes 43 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 3856023 bytes
> Compression is 0.0%
> Took 1.5 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 730 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/730/

1 tests failed.
REGRESSION:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField

Error Message:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/index/_az_TestBloomFilteredLucenePostings_0.tim: Too many open files
	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
	at java.nio.file.Files.newOutputStream(Files.java:172)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter.<init>(BlockTreeTermsWriter.java:278)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsConsumer(Lucene50PostingsFormat.java:433)
	at org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat.fieldsConsumer(BloomFilteringPostingsFormat.java:147)
	at org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings.fieldsConsumer(TestBloomFilteredLucenePostings.java:66)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:196)
	at org.apache.lucene.index.FreqProxTermsWriter.flush(FreqProxTermsWriter.java:107)
	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:112)
	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
	at org.apache.lucene.index.DocumentsWriter.postUpdate(DocumentsWriter.java:373)
	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:466)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1076)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 1862 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDemoParallelLeafReader
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #1,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyvp_40/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #12,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz6a_43/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #6,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyzj_43/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #16,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzbm_45/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #14,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz93_44/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #4,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyxe_41/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #2,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyw4_40/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #5,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyz5_42/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #7,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpyww_43/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #20,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzjj_45/_0.fdt: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]   2> 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #21,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzll_45: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
   [junit4]   2> 	at java.nio.file.Files.newDirectoryStream(Files.java:545)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
   [junit4]   2> 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #19,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzge_45: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
   [junit4]   2> 	at java.nio.file.Files.newDirectoryStream(Files.java:545)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
   [junit4]   2> 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #0,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpwp6_40/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #13,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz70_43: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
   [junit4]   2> 	at java.nio.file.Files.newDirectoryStream(Files.java:545)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
   [junit4]   2> 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #8,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz1o_43: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
   [junit4]   2> 	at java.nio.file.Files.newDirectoryStream(Files.java:545)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
   [junit4]   2> 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #15,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz9t_45/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #3,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpywn_41: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newDirectoryStream(HandleTrackingFS.java:298)
   [junit4]   2> 	at java.nio.file.Files.newDirectoryStream(Files.java:545)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:171)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.listAll(FSDirectory.java:191)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.listAll(MockDirectoryWrapper.java:1000)
   [junit4]   2> 	at org.apache.lucene.index.DirectoryReader.indexExists(DirectoryReader.java:288)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:787)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:823)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #11,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz52_43/_0_Memory_0.mdvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:76)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #10,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpz3l_43/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1678)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.forceMerge(IndexWriter.java:1658)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:851)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:15 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #18,5,TGRP-TestDemoParallelLeafReader]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([78910BB799798171]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:611)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:581)
   [junit4]   2> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001/tempDir-002/segs/6za9dzr2yvnqeaarom3jvpzem_45/_0_Memory_0.mdvm: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesConsumer.<init>(MemoryDocValuesConsumer.java:79)
   [junit4]   2> 	at org.apache.lucene.codecs.memory.MemoryDocValuesFormat.fieldsConsumer(MemoryDocValuesFormat.java:60)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.getInstance(PerFieldDocValuesFormat.java:187)
   [junit4]   2> 	at org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsWriter.addNumericField(PerFieldDocValuesFormat.java:111)
   [junit4]   2> 	at org.apache.lucene.index.NumericDocValuesWriter.flush(NumericDocValuesWriter.java:90)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.writeDocValues(DefaultIndexingChain.java:142)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.flush(DefaultIndexingChain.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:419)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:503)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:615)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3019)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:2994)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.shutdown(IndexWriter.java:963)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.close(IndexWriter.java:1008)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:854)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]   2> 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> 
   [junit4]   2> jan 10, 2015 7:12:20 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception 

[...truncated too long message...]

Wrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:105)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:179)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingStoredFieldsFormat.fieldsWriter(AssertingStoredFieldsFormat.java:49)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:258)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:295)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:449)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1415)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1150)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1135)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$3.reindex(TestDemoParallelLeafReader.java:835)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getParallelLeafReader(TestDemoParallelLeafReader.java:397)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.getCurrentReader(TestDemoParallelLeafReader.java:161)
   [junit4]    > 	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader$ReindexingMergePolicy$ReindexingOneMerge.getMergeReaders(TestDemoParallelLeafReader.java:535)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3958)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3559)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:519)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:557)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.index.TestDemoParallelLeafReader 78910BB799798171-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), number=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{number_1=DocValuesFormat(name=Lucene50), number_4=DocValuesFormat(name=Asserting), number_53=DocValuesFormat(name=Asserting), number_84=DocValuesFormat(name=Lucene50), number_78=DocValuesFormat(name=Lucene50), number_83=DocValuesFormat(name=Memory), number_71=DocValuesFormat(name=Asserting), number_0=DocValuesFormat(name=Asserting), number_87=DocValuesFormat(name=Memory), number_30=DocValuesFormat(name=Lucene50), number_67=DocValuesFormat(name=Lucene50), number_57=DocValuesFormat(name=Asserting), number_20=DocValuesFormat(name=Asserting), number_69=DocValuesFormat(name=Memory), number_59=DocValuesFormat(name=Lucene50), number_73=DocValuesFormat(name=Lucene50), number_77=DocValuesFormat(name=Lucene50), number_175=DocValuesFormat(name=Memory), number_130=DocValuesFormat(name=Lucene50), number_49=DocValuesFormat(name=Lucene50), number_88=DocValuesFormat(name=Lucene50), number_70=DocValuesFormat(name=Lucene50), number_82=DocValuesFormat(name=Asserting), number_86=DocValuesFormat(name=Asserting), number_3=DocValuesFormat(name=Memory), number_92=DocValuesFormat(name=Lucene50), number_74=DocValuesFormat(name=Lucene50), number_198=DocValuesFormat(name=Asserting), number_160=DocValuesFormat(name=Memory), number_68=DocValuesFormat(name=Asserting), number_75=DocValuesFormat(name=Asserting), number_7=DocValuesFormat(name=Memory), number_76=DocValuesFormat(name=Memory), number_79=DocValuesFormat(name=Asserting), number_2=DocValuesFormat(name=Lucene50), number_72=DocValuesFormat(name=Memory), number_22=DocValuesFormat(name=Lucene50), number_99=DocValuesFormat(name=Lucene50), number_90=DocValuesFormat(name=Memory), number_66=DocValuesFormat(name=Lucene50), number_80=DocValuesFormat(name=Lucene50), number_91=DocValuesFormat(name=Lucene50), number=DocValuesFormat(name=Memory), number_81=DocValuesFormat(name=Lucene50), number_42=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=sk, timezone=Asia/Ashgabat
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=138795984,total=459276288
   [junit4]   2> NOTE: All tests run in this JVM: [TestTotalHitCountCollector, TestPhraseQuery, TestDirectoryReaderReopen, TestSimpleAttributeImpl, TestBasics, TestRegExp, TestSnapshotDeletionPolicy, TestBoolean2, Nested1, TestRoaringDocIdSet, Test2BSortedDocValues, TestPositiveScoresOnlyCollector, TestUnicodeUtil, TestPerFieldPostingsFormat2, TestWildcard, TestLucene50NormsFormat, TestBitDocIdSetBuilder, TestDocsAndPositions, Nested1, TestBagOfPostings, TestPayloadNearQuery, TestFilteredQuery, TestExternalCodecs, TestIndexWriterConfig, TestFieldCacheTermsFilter, TestWorstCaseTestBehavior, TestFlex, TestPersistentSnapshotDeletionPolicy, TestCachingWrapperFilter, Test2BTerms, TestMultiTermQueryRewrites, TestSpanExplanationsOfNonMatches, TestBinaryTerms, TestSpanSearchEquivalence, TestBufferedIndexInput, TestMultiLevelSkipList, TestSimilarityBase, TestDocInverterPerFieldErrorInfo, TestAssertingDocValuesFormat, TestStringHelper, TestBinaryDocument, TestDeterminizeLexicon, TestIndexWriterWithThreads, TestSpansAdvanced, TestWindowsMMap, TestSearch, Nested1, TestNeverDelete, TestDocIdSetBuilder, TestNoDeletionPolicy, TestIndexWriterOnJRECrash, TestIndexWriterReader, TestCompressingTermVectorsFormat, TestIndexWriterMerging, TestFSTs, TestPackedInts, TestMultiMMap, TestFlushByRamOrCountsPolicy, TestSloppyPhraseQuery, TestIndexWriterDelete, TestNorms, TestSearchWithThreads, TestSearchAfter, TestConcurrentMergeScheduler, TestHighCompressionMode, TestDocumentsWriterStallControl, TestTopDocsMerge, TestStressIndexing, TestMinimize, TestRollingUpdates, TestNRTReaderWithThreads, TestPerFieldPostingsFormat, TestConsistentFieldNumbers, TestSimpleExplanations, TestDocTermOrdsRangeFilter, TestCompressingStoredFieldsFormat, TestSegmentMerger, TestBagOfPositions, TestFixedBitSet, TestNRTCachingDirectory, TestIndexWriterNRTIsCurrent, TestPostingsOffsets, TestMixedCodecs, TestTransactionRollback, TestUTF32ToUTF8, TestSpanExplanations, TestStressIndexing2, TestMultiPhraseQuery, TestLazyProxSkipping, TestDeterminism, TestCodecs, TestCharTermAttributeImpl, TestOmitPositions, TestPayloadExplanations, TestParallelLeafReader, TestBytesRefHash, TestThreadedForceMerge, TestIndexInput, TestSloppyPhraseQuery2, TestDisjunctionMaxQuery, TestForceMergeForever, TestSameScoresWithThreads, TestQueryWrapperFilter, TestMultiFields, TestSimpleExplanationsOfNonMatches, TestTermVectorsReader, TestSpansAdvanced2, TestIntBlockPool, TestPrefixInBooleanQuery, TestTopDocsCollector, TestSegmentTermEnum, TestUniqueTermCount, TestBooleanQuery, TestSortRandom, TestPrefixCodedTerms, TestAllFilesHaveCodecHeader, TestDocument, TestTermScorer, TestTopFieldCollector, TestSumDocFreq, TestIndexableField, TestIndexFileDeleter, TestMatchAllDocsQuery, TestParallelTermEnum, TestSubScorerFreqs, Nested, Nested, TestDateSort, TestVirtualMethod, TestDateTools, TestAutomatonQueryUnicode, TestMockDirectoryWrapper, TestNoMergePolicy, TestNamedSPILoader, TestBytesRef, TestNot, TestVersion, TestScoreCachingWrappingScorer, TestTermdocPerf, TestIndexCommit, TestNewestSegment, Test2BBinaryDocValues, TestReusableStringReader, TestBlockPostingsFormat, TestForUtil, TestLucene50CompoundFormat, TestFieldType, TestAllFilesHaveChecksumFooter, TestCodecUtil, TestDemoParallelLeafReader]
   [junit4] Completed on J3 in 610.62s, 5 tests, 1 error <<< FAILURES!

[...truncated 15 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 409 suites, 3237 tests, 1 error, 37 ignored (27 assumptions)

Total time: 39 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 3856023 bytes
Compression is 0.0%
Took 1.5 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 729 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/729/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:192)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:413)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:143)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63800/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63800/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63800/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 46 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24139/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24139/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:75)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:48)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24139/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:63139/l/l/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63139/l/l/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:137)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60834/ho/lx/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60834/ho/lx/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:581)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:890)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60834/ho/lx/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:578)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9848 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/init-core-data-001
   [junit4]   2> 356608 T1093 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ho/lx
   [junit4]   2> 356616 T1093 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 356617 T1093 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 356618 T1094 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 356718 T1093 oasc.ZkTestServer.run start zk server on port:60800
   [junit4]   2> 356719 T1093 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 356720 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 356725 T1101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6125b19 name:ZooKeeperConnection Watcher:127.0.0.1:60800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 356725 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 356726 T1093 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 356726 T1093 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 356729 T1093 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 356730 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 356738 T1104 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f322dd7 name:ZooKeeperConnection Watcher:127.0.0.1:60800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 356739 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 356739 T1093 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 356740 T1093 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 356742 T1093 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 356744 T1093 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 356746 T1093 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 356748 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 356749 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 356752 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 356753 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 356756 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 356757 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 356759 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 356760 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 356762 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 356763 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 356765 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 356766 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 356768 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 356769 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 356771 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 356772 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 356774 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 356775 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 356777 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 356778 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 356780 T1093 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 356780 T1093 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 358234 T1093 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 358238 T1093 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60806
   [junit4]   2> 358239 T1093 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 358239 T1093 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 358240 T1093 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002
   [junit4]   2> 358240 T1093 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002/'
   [junit4]   2> 358287 T1093 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002/solr.xml
   [junit4]   2> 358379 T1093 oasc.CoreContainer.<init> New CoreContainer 911598027
   [junit4]   2> 358380 T1093 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002/]
   [junit4]   2> 358381 T1093 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 358382 T1093 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 358382 T1093 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 358382 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 358383 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 358383 T1093 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 358383 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 358384 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 358384 T1093 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 358384 T1093 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 358385 T1093 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 358386 T1093 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 358386 T1093 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 358386 T1093 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 358387 T1093 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60800/solr
   [junit4]   2> 358387 T1093 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 358388 T1093 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 358389 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 358391 T1116 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18571081 name:ZooKeeperConnection Watcher:127.0.0.1:60800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 358392 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 358393 T1093 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 358395 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 358397 T1119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@733e58ab name:ZooKeeperConnection Watcher:127.0.0.1:60800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 358397 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 358399 T1093 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 358402 T1093 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 358404 T1093 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 358406 T1093 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 358409 T1093 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 358411 T1093 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 358412 T1093 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60805_ho%2Flx
   [junit4]   2> 358413 T1093 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60805_ho%2Flx
   [junit4]   2> 358415 T1093 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 358416 T1093 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 358418 T1093 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 358420 T1093 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60805_ho%2Flx
   [junit4]   2> 358420 T1093 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 358422 T1093 oasc.Overseer.start Overseer (id=93114684115582979-127.0.0.1:60805_ho%2Flx-n_0000000000) starting
   [junit4]   2> 358424 T1093 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 358430 T1093 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 358431 T1121 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 358432 T1093 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 358434 T1093 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 358435 T1093 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 358438 T1120 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 358443 T1123 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 358444 T1123 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 358445 T1123 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 358445 T1119 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 358448 T1120 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:60805_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60805/ho/lx",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 358448 T1120 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:60805_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60805/ho/lx",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 358449 T1120 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 358449 T1120 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 358450 T1119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 359445 T1123 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 359446 T1123 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 359447 T1123 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 359447 T1123 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 359447 T1123 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 359448 T1123 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002/collection1/'
   [junit4]   2> 359449 T1123 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 359450 T1123 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 359451 T1123 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 359488 T1123 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 359491 T1123 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 359504 T1123 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 359527 T1123 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 359590 T1123 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 359591 T1123 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 359608 T1123 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 360006 T1123 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 360019 T1123 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 360022 T1123 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 360036 T1123 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360040 T1123 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360045 T1123 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360046 T1123 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360047 T1123 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360047 T1123 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360048 T1123 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360048 T1123 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360048 T1123 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 360049 T1123 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 360049 T1123 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/control/data/
   [junit4]   2> 360049 T1123 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ffd0d9d
   [junit4]   2> 360050 T1123 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/control/data
   [junit4]   2> 360050 T1123 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/control/data/index/
   [junit4]   2> 360050 T1123 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 360051 T1123 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/control/data/index
   [junit4]   2> 360051 T1123 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4fe20db9
   [junit4]   2> 360052 T1123 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@1f315a8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e12c997; maxCacheMB=0.8564453125 maxMergeSizeMB=0.767578125)),segFN=segments_1,generation=1}
   [junit4]   2> 360053 T1123 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360057 T1123 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 360057 T1123 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 360058 T1123 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 360058 T1123 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360058 T1123 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360058 T1123 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 360059 T1123 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360059 T1123 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360059 T1123 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 360060 T1123 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 360061 T1123 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 360061 T1123 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 360061 T1123 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 360062 T1123 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 360062 T1123 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 360063 T1123 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 360063 T1123 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 360064 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 360065 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 360065 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 360066 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 360066 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 360067 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 360067 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 360068 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 360068 T1123 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 360069 T1123 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 360095 T1123 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360098 T1123 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360100 T1123 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360102 T1123 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360105 T1123 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 360109 T1123 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 360109 T1123 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 360110 T1123 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=1.904296875, floorSegmentMB=1.5078125, forceMergeDeletesPctAllowed=21.87901865227614, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42311695926325554
   [junit4]   2> 360111 T1123 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@1f315a8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e12c997; maxCacheMB=0.8564453125 maxMergeSizeMB=0.767578125)),segFN=segments_1,generation=1}
   [junit4]   2> 360111 T1123 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360111 T1123 oass.SolrIndexSearcher.<init> Opening Searcher@1ce674d0[collection1] main
   [junit4]   2> 360112 T1123 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 360113 T1123 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 360113 T1123 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 360113 T1123 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 360114 T1123 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 360114 T1123 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360114 T1123 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 360115 T1123 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360115 T1123 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 360115 T1123 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 360116 T1123 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 360116 T1124 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ce674d0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 360116 T1127 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60805/ho/lx collection:control_collection shard:shard1
   [junit4]   2> 360117 T1093 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 360117 T1093 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 360117 T1127 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 360121 T1093 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 360121 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 360122 T1127 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 360124 T1131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54bec9bb name:ZooKeeperConnection Watcher:127.0.0.1:60800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 360124 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 360124 T1093 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 360124 T1119 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 360125 T1127 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 360126 T1127 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C249 name=collection1 org.apache.solr.core.SolrCore@46f780e2 url=http://127.0.0.1:60805/ho/lx/collection1 node=127.0.0.1:60805_ho%2Flx C249_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:60805_ho%2Flx, base_url=http://127.0.0.1:60805/ho/lx, state=down, core=collection1}
   [junit4]   2> 360126 T1127 C249 P60805 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60805/ho/lx/collection1/
   [junit4]   2> 360126 T1120 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 360126 T1127 C249 P60805 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 360126 T1093 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 360127 T1127 C249 P60805 oasc.SyncStrategy.syncToMe http://127.0.0.1:60805/ho/lx/collection1/ has no replicas
   [junit4]   2> 360127 T1127 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60805/ho/lx/collection1/ shard1
   [junit4]   2> 360127 T1127 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 360128 T1119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 360128 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 360130 T1093 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 360132 T1132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 360132 T1120 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60805/ho/lx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 360235 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 360284 T1127 oasc.ZkController.register We are http://127.0.0.1:60805/ho/lx/collection1/ and leader is http://127.0.0.1:60805/ho/lx/collection1/
   [junit4]   2> 360285 T1127 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60805/ho/lx
   [junit4]   2> 360285 T1127 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 360285 T1127 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 360285 T1127 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 360287 T1119 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 360288 T1120 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:60805_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60805/ho/lx",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 360289 T1120 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:60805_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60805/ho/lx",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 360392 T1119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 360392 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 361340 T1093 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 361341 T1093 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 361343 T1093 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60812
   [junit4]   2> 361344 T1093 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 361344 T1093 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 361344 T1093 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003
   [junit4]   2> 361345 T1093 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003/'
   [junit4]   2> 361384 T1093 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003/solr.xml
   [junit4]   2> 361443 T1093 oasc.CoreContainer.<init> New CoreContainer 2108549278
   [junit4]   2> 361444 T1093 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003/]
   [junit4]   2> 361444 T1093 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 361445 T1093 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 361445 T1093 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 361445 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 361445 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 361446 T1093 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 361446 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 361446 T1093 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 361446 T1093 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 361446 T1093 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 361447 T1093 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 361447 T1093 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 361447 T1093 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 361448 T1093 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 361448 T1093 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:60800/solr
   [junit4]   2> 361448 T1093 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 361448 T1093 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 361449 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 361451 T1144 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d26b094 name:ZooKeeperConnection Watcher:127.0.0.1:60800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 361451 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 361451 T1093 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 361453 T1093 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 361454 T1147 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@651515a name:ZooKeeperConnection Watcher:127.0.0.1:60800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 361454 T1093 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 361458 T1093 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 362461 T1093 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60811_ho%2Flx
   [junit4]   2> 362462 T1093 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60811_ho%2Flx
   [junit4]   2> 362464 T1093 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 362470 T1148 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 362470 T1148 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 362470 T1148 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 362470 T1119 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 362471 T1120 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:60811_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60811/ho/lx",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 362472 T1120 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:60811_ho%2Flx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60811/ho/lx",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 362472 T1120 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 362472 T1120 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 362473 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 362473 T1119 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 362473 T1147 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 363471 T1148 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 363472 T1148 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 363472 T1148 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 363473 T1148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 363473 T1148 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 363473 T1148 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003/collection1/'
   [junit4]   2> 363475 T1148 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 363476 T1148 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 363476 T1148 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest%201E06DB145A0CCFC6-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 363510 T1148 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 363512 T1148 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 363525 T1148 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 363548 T1148 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 363607 T1148 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 363608 T1148 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 363617 T1148 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 363998 T1148 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 364011 T1148 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 364014 T1148 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 364027 T1148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 364031 T1148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 364035 T1148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 364036 T1148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 364037 T1148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 364037 T1148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 364038 T1148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 364038 T1148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 364038 T1148 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 364039 T1148 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 364039 T1148 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/jetty1/
   [junit4]   2> 364039 T1148 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1ffd0d9d
   [junit4]   2> 364040 T1148 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/jetty1
   [junit4]   2> 364040 T1148 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/jetty1/index/
   [junit4]   2> 364040 T1148 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 364041 T1148 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1E06DB145A0CCFC6-001/tempDir-001/jetty1/index
   [junit4]   2> 364041 T1148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@34a31f43
   [junit4]   2> 364042 T1148 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@415982b4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b9418f0; maxCacheMB=0.8564453125 maxMergeSizeMB=0.767578125)),segFN=segments_1,generation=1}
   [junit4]   2> 364042 T1148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 364048 T1148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 364048 T1148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 364049 T1148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 364049 T1148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 364049 T1148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 364049 T1148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 364050 T1148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 364050 T1148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 364050 T1148 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 364051 T1148 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 364052 T1148 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 364052 T1148 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 364053 T1148 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 364053 T1148 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 364054 T1148 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 364055 T1148 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 364056 T1148 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 364056 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 364057 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 364058 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 364058 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 364059 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 364059 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 364060 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 364061 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 364061 T1148 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 364062 T1148 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 364084 T1148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 364086 T1148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 364088 T1148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 364090 T1148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 364092 T1148 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 364095 T1148 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 364095 T1148 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 364096 T1148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=1.904296875, floorSegmentMB=1.5078125, forceMergeDeletesPctAllowed=21.87901865227614, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42311695926325554
   [junit4]   2> 364097 T1148 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@415982b4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b9418f0; maxCacheMB=0.8564453125 maxMergeSizeMB=0.767578125)),segFN=segments_1,generation=1}
   [junit4]   2> 364097 T1148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 364098 T1148 oass.SolrIndexSearcher.<init> Opening Searcher@7e12b7be[collection1] main
   [junit4]   2> 364098 T1148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 364099 T1148 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 364099 T1148 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 364099 T1148 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 364100 T1148 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 364100 T1148 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 364100 T1148 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 364101 T1148 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 364101 T1148 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 364101 T1148 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 364102 T1149 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7e12b7be[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 364103 T1148 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 364103 T11

[...truncated too long message...]

:135)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 2295166 T4971 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2295168 T4841 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/l/l,null}
   [junit4]   2> 2295363 T4841 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2295365 T4841 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63098 63098
   [junit4]   2> 2296416 T4842 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63098 63098
   [junit4]   2> 2296417 T4842 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93114806455828492-core_node2-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 2296418 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63161/l/l
   [junit4]   2> 2296418 T4841 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:63148/l/l
   [junit4]   2> 2296419 T4841 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:63139/l/l
   [junit4]   2> 2296419 T4841 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:63129/l/l
   [junit4]   2> 2296419 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63107/l/l
   [junit4]   2> 2296420 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63161/l/l
   [junit4]   2> 2296420 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63148/l/l
   [junit4]   2> 2296421 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63139/l/l
   [junit4]   2> 2296421 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63129/l/l
   [junit4]   2> 2296421 T4841 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63107/l/l
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=testDistribSearch -Dtests.seed=1E06DB145A0CCFC6 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_KW -Dtests.timezone=CST -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.7s J2 | LeaderInitiatedRecoveryOnCommitTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63139/l/l/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1E06DB145A0CCFC6:9FE0550C2D53AFFA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:137)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 48 more
   [junit4]   2> 2296433 T4841 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 1E06DB145A0CCFC6-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_KW, timezone=CST
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=118249304,total=285212672
   [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, RollingRestartTest, TestReplicationHandler, DocumentBuilderTest, CachingDirectoryFactoryTest, SolrRequestParserTest, ReturnFieldsTest, SolrPluginUtilsTest, SimpleMLTQParserTest, TestFreeTextSuggestions, TestSchemaSimilarityResource, EchoParamsTest, QueryEqualityTest, TestOrdValues, StatsComponentTest, TestSolrDeletionPolicy2, BadCopyFieldTest, TestReversedWildcardFilterFactory, SpellCheckComponentTest, NoCacheHeaderTest, TestAnalyzedSuggestions, TestIntervalFaceting, LeaderFailoverAfterPartitionTest, QueryResultKeyTest, TestCSVResponseWriter, TestHighFrequencyDictionaryFactory, TestReload, SolrXmlInZkTest, TestManagedResourceStorage, AddSchemaFieldsUpdateProcessorFactoryTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 76.73s, 1 test, 1 error <<< FAILURES!

[...truncated 294 lines...]
   [junit4] JVM J1: stdout was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/temp/junit4-J1-20150109_152242_142.sysout
   [junit4] >>> JVM J1: stdout (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/heapdumps/java_pid29706.hprof ...
   [junit4] Heap dump file created [503187398 bytes in 6.953 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 28 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 459 suites, 1869 tests, 5 errors, 57 ignored (10 assumptions)

Total time: 146 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 6 artifacts
Archive block size is 32768
Received 0 blocks and 1181649022 bytes
Compression is 0.0%
Took 4 min 22 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 728 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/728/

7 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:24999/_jc/dv: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:532)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:255)  at org.apache.solr.cloud.ZkController.register(ZkController.java:901)  at org.apache.solr.cloud.ZkController.register(ZkController.java:803)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:465)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:518)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:485)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:738)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:254)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_jc/dv/admin/cores. Reason: <pre>    {trace=java.lang.AssertionError  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:532)  at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:255)  at org.apache.solr.cloud.ZkController.register(ZkController.java:901)  at org.apache.solr.cloud.ZkController.register(ZkController.java:803)  at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)  at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)  at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:465)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:518)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:485)  at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)  at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:738)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:254)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) ,code=500}</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:24999/_jc/dv: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:532)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:255)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:901)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:803)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:465)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:518)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:485)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:738)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:254)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /_jc/dv/admin/cores. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:532)
	at org.apache.solr.common.cloud.ZkStateReader.updateClusterState(ZkStateReader.java:255)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:901)
	at org.apache.solr.cloud.ZkController.register(ZkController.java:803)
	at org.apache.solr.core.ZkContainer$2.run(ZkContainer.java:208)
	at org.apache.solr.core.ZkContainer.registerInZk(ZkContainer.java:232)
	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:465)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:518)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:485)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:738)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:254)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:201)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:527)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:449)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:201)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:201)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:114)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:75)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:48)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62278/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62278/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:169)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:147)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:137)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.doTest(LeaderInitiatedRecoveryOnCommitTest.java:70)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 305 from http://127.0.0.1:25474/collection1lastClient and got 5 from http://127.0.0.1:25480/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 305 from http://127.0.0.1:25474/collection1lastClient and got 5 from http://127.0.0.1:25480/collection1
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1250)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1229)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:235)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestCloudPivotFacet.testDistribSearch

Error Message:
{main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti1&facet.pivot=%7B%21stats%3Dst3%7Dpivot_tdt1&facet.limit=12&facet.sort=index&facet.overrequest.count=0&facet.overrequest.ratio=0),extra(rows=0&q=*%3A*&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tl1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_tl&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_f1&_test_sort=index)} ==> Sum of sk3 => pivot_tdt1: {params(rows=0),defaults({main(rows=0&q=*%3A*&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tl1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_tl&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_f1&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_tdt1%7D2010-10-31T10%3A31%3A04Z)})} expected:<5.1523870735876626E8> but was:<5.152387073587663E8>

Stack Trace:
java.lang.AssertionError: {main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti1&facet.pivot=%7B%21stats%3Dst3%7Dpivot_tdt1&facet.limit=12&facet.sort=index&facet.overrequest.count=0&facet.overrequest.ratio=0),extra(rows=0&q=*%3A*&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tl1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_tl&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_f1&_test_sort=index)} ==> Sum of sk3 => pivot_tdt1: {params(rows=0),defaults({main(rows=0&q=*%3A*&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tl1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_tl&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_f1&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_tdt1%7D2010-10-31T10%3A31%3A04Z)})} expected:<5.1523870735876626E8> but was:<5.152387073587663E8>
	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:276)
	at org.apache.solr.cloud.TestCloudPivotFacet.doTest(TestCloudPivotFacet.java:223)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: Sum of sk3 => pivot_tdt1: {params(rows=0),defaults({main(rows=0&q=*%3A*&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tl1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_tl&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_f1&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_tdt1%7D2010-10-31T10%3A31%3A04Z)})} expected:<5.1523870735876626E8> but was:<5.152387073587663E8>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotStats(TestCloudPivotFacet.java:379)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:334)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:297)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:266)
	... 42 more




Build Log:
[...truncated 9420 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/init-core-data-001
   [junit4]   2> 25600 T27 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_ero/pa
   [junit4]   2> 25621 T27 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 25679 T27 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 25696 T28 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 26198 T27 oasc.ZkTestServer.run start zk server on port:34329
   [junit4]   2> 26215 T27 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 26615 T27 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 26900 T35 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13cd4abe name:ZooKeeperConnection Watcher:127.0.0.1:34329 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 26901 T27 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 26901 T27 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 26923 T27 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 27141 T27 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 27142 T27 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27145 T38 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ac11f52 name:ZooKeeperConnection Watcher:127.0.0.1:34329/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27146 T27 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27146 T27 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 27178 T27 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 27183 T27 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 27188 T27 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 27192 T27 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 27198 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 27200 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 27208 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 27209 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 27215 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 27216 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 27222 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 27223 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 27228 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 27230 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 27235 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 27236 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 27242 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 27243 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 27248 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 27250 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 27255 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 27257 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 27262 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 27264 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 27269 T27 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 27271 T27 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 29362 T27 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 29367 T27 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35440
   [junit4]   2> 29369 T27 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 29370 T27 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 29371 T27 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002
   [junit4]   2> 29372 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002/'
   [junit4]   2> 29452 T27 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002/solr.xml
   [junit4]   2> 29630 T27 oasc.CoreContainer.<init> New CoreContainer 316267329
   [junit4]   2> 29631 T27 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002/]
   [junit4]   2> 29633 T27 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 29634 T27 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 29635 T27 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 29636 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 29637 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 29638 T27 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 29639 T27 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 29639 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 29640 T27 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 29641 T27 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 29643 T27 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 29644 T27 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 29644 T27 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 29645 T27 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 29646 T27 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34329/solr
   [junit4]   2> 29739 T27 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 29740 T27 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 29742 T27 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29747 T50 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bca8054 name:ZooKeeperConnection Watcher:127.0.0.1:34329 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29748 T27 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29748 T27 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 29792 T27 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29796 T53 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@305460f6 name:ZooKeeperConnection Watcher:127.0.0.1:34329/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29796 T27 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29829 T27 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 29857 T27 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 29874 T27 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 29881 T27 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 29888 T27 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 29919 T27 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 29923 T27 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22961__ero%2Fpa
   [junit4]   2> 29934 T27 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22961__ero%2Fpa
   [junit4]   2> 29941 T27 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 29945 T27 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 29951 T27 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 29989 T27 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22961__ero%2Fpa
   [junit4]   2> 29991 T27 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 29997 T27 oasc.Overseer.start Overseer (id=93108979062603779-127.0.0.1:22961__ero%2Fpa-n_0000000000) starting
   [junit4]   2> 30043 T27 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 30107 T27 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 30906 T55 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 30907 T27 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 30911 T27 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 30913 T27 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 30958 T54 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 31002 T57 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 31003 T57 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 31026 T53 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 31026 T57 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 31043 T54 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22961__ero%2Fpa",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22961/_ero/pa",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 31133 T54 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22961__ero%2Fpa",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22961/_ero/pa",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 31144 T54 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 31217 T54 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 31239 T53 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 32028 T57 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 32028 T57 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 32030 T57 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 32031 T57 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 32033 T57 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 32034 T57 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002/collection1/'
   [junit4]   2> 32037 T57 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest%20F5EA07367083508A-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 32038 T57 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest%20F5EA07367083508A-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 32040 T57 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest%20F5EA07367083508A-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 32158 T57 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 32208 T57 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 32290 T57 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 32339 T57 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 32429 T57 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 32432 T57 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 32541 T57 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 34312 T57 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34352 T57 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34355 T57 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34551 T57 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34576 T57 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34583 T57 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34590 T57 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34590 T57 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34591 T57 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34593 T57 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34593 T57 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34594 T57 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 34594 T57 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34595 T57 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-001/control/data/
   [junit4]   2> 34604 T57 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d4390ac
   [junit4]   2> 34704 T57 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-001/control/data
   [junit4]   2> 34705 T57 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-001/control/data/index/
   [junit4]   2> 34705 T57 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 34753 T57 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/tempDir-001/control/data/index
   [junit4]   2> 34754 T57 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=3, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=76.6025390625, floorSegmentMB=0.810546875, forceMergeDeletesPctAllowed=5.9348069863423625, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6197460427205737
   [junit4]   2> 34774 T57 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b3bb7fb),segFN=segments_1,generation=1}
   [junit4]   2> 34774 T57 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34784 T57 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 35009 T57 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 35020 T57 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 35021 T57 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35021 T57 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35021 T57 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35248 T57 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35248 T57 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35249 T57 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35257 T57 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35258 T57 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35260 T57 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35261 T57 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35262 T57 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 35263 T57 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 35265 T57 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 35266 T57 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 35267 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 35269 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 35271 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 35272 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 35273 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 35274 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 35276 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 35277 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 35278 T57 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 35334 T57 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 35354 T57 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35356 T57 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35359 T57 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35361 T57 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35370 T57 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35470 T57 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35470 T57 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35472 T57 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=72.416015625, floorSegmentMB=0.4521484375, forceMergeDeletesPctAllowed=18.25353016828899, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 35527 T57 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest F5EA07367083508A-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b3bb7fb),segFN=segments_1,generation=1}
   [junit4]   2> 35528 T57 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 35528 T57 oass.SolrIndexSearcher.<init> Opening Searcher@29669b01[collection1] main
   [junit4]   2> 35529 T57 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 35531 T57 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 35532 T57 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 35547 T57 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 35548 T57 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 35548 T57 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 35550 T57 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 35550 T57 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 35551 T57 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35581 T57 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 35586 T58 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29669b01[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())

[...truncated too long message...]

g.apache.solr.core.SolrCore@4e22811e url=http://127.0.0.1:25480/collection1 node=127.0.0.1:25480_ C973_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:25480, core=collection1, node_name=127.0.0.1:25480_, state=active}
   [junit4]   2> 2528738 T4636 C973 P25480 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 2528739 T4636 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 2528739 T4636 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e22811e
   [junit4]   2> 2528741 T4636 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 2528742 T4636 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=29,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=654,transaction_logs_total_number=2}
   [junit4]   2> 2528742 T4636 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2528743 T4636 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2528743 T4636 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2528744 T4636 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2528746 T4636 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2528746 T4636 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7;done=false>>]
   [junit4]   2> 2528746 T4636 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7
   [junit4]   2> 2528746 T4636 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7/index;done=false>>]
   [junit4]   2> 2528747 T4636 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001/tempDir-001/jetty7/index
   [junit4]   2> 2528747 T4164 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2528949 T4164 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2528951 T4164 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:25199 25199
   [junit4]   2> 2529058 T4165 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:25199 25199
   [junit4]   2> 2529060 T4165 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		10	/solr/aliases.json
   [junit4]   2> 		10	/solr/clusterstate.json
   [junit4]   2> 		9	/solr/configs/conf1
   [junit4]   2> 		8	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		3	/solr/overseer_elect/election/93109040540811267-127.0.0.1:25220_-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		9	/solr/live_nodes
   [junit4]   2> 		6	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=F5EA07367083508A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_PR -Dtests.timezone=Asia/Bangkok -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 1565s J2 | SyncSliceTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 305 from http://127.0.0.1:25474/collection1lastClient and got 5 from http://127.0.0.1:25480/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F5EA07367083508A:740C892E07DC30B6]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1250)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1229)
   [junit4]    > 	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:235)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2529070 T4164 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.SyncSliceTest F5EA07367083508A-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=MockRandom), a_i1=BlockTreeOrds(blocksize=128), id=FST50, text=PostingsFormat(name=Memory doPackFST= true), timestamp=BlockTreeOrds(blocksize=128), rnd_b=FST50, multiDefault=BlockTreeOrds(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), a_t=BlockTreeOrds(blocksize=128), _version_=PostingsFormat(name=MockRandom), other_tl1=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=es_PR, timezone=Asia/Bangkok
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=246758712,total=383778816
   [junit4]   2> NOTE: All tests run in this JVM: [TestElisionMultitermQuery, DeleteInactiveReplicaTest, NoCacheHeaderTest, TestRealTimeGet, StandardRequestHandlerTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, PrimUtilsTest, TestDefaultSimilarityFactory, DocumentBuilderTest, TestTolerantSearch, SimpleMLTQParserTest, PeerSyncTest, TestCloudPivotFacet, TestFileDictionaryLookup, ScriptEngineTest, TestSolrConfigHandlerConcurrent, HdfsDirectoryTest, TestDistributedSearch, TestLazyCores, TestSchemaSimilarityResource, TestConfig, MergeStrategyTest, TestSolrQueryParserResource, DirectUpdateHandlerOptimizeTest, EchoParamsTest, SolrCoreCheckLockOnStartupTest, TestAnalyzedSuggestions, TestReversedWildcardFilterFactory, TestCSVResponseWriter, TestPHPSerializedResponseWriter, TestCloudInspectUtil, TestFoldingMultitermQuery, TestFreeTextSuggestions, MultiThreadedOCPTest, TestLMDirichletSimilarityFactory, CursorMarkTest, TestDocumentBuilder, HighlighterConfigTest, SolrPluginUtilsTest, SolrIndexSplitterTest, TestExtendedDismaxParser, CurrencyFieldOpenExchangeTest, RemoteQueryErrorTest, TestRemoteStreaming, ReturnFieldsTest, SharedFSAutoReplicaFailoverTest, CacheHeaderTest, AlternateDirectoryTest, TestManagedResourceStorage, TestHashPartitioner, SpellCheckComponentTest, TestReRankQParserPlugin, CustomCollectionTest, SolrXmlInZkTest, DirectUpdateHandlerTest, SyncSliceTest]
   [junit4] Completed on J2 in 1564.71s, 1 test, 1 failure <<< FAILURES!

[...truncated 276 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 459 suites, 1867 tests, 5 errors, 2 failures, 57 ignored (10 assumptions)

Total time: 149 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 497186419 bytes
Compression is 0.0%
Took 1 min 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 727 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/727/

2 tests failed.
REGRESSION:  org.apache.lucene.index.Test4GBStoredFields.test

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([A2D310980D63ED69]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.Test4GBStoredFields

Error Message:
Suite timeout exceeded (>= 14400000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 14400000 msec).
	at __randomizedtesting.SeedInfo.seed([A2D310980D63ED69]:0)




Build Log:
[...truncated 1942 lines...]
   [junit4] Suite: org.apache.lucene.index.Test4GBStoredFields
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=Test4GBStoredFields -Dtests.method=test -Dtests.seed=A2D310980D63ED69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Pacific/Guadalcanal -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   14400s J0 | Test4GBStoredFields.test <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A2D310980D63ED69]:0)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J0/temp/lucene.index.Test4GBStoredFields A2D310980D63ED69-001
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, maxDocsPerChunk=920, blockSize=9), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=1, blockSize=9)), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=en_ZA, timezone=Pacific/Guadalcanal
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=228253960,total=454033408
   [junit4]   2> NOTE: All tests run in this JVM: [TestDisjunctionMaxQuery, TestBytesStore, TestLazyProxSkipping, TestLevenshteinAutomata, TestIndexInput, TestSimpleSearchEquivalence, TestAssertingTermVectorsFormat, TestAutomatonQuery, TestCachingTokenFilter, TestNotDocIdSet, TestMatchAllDocsQuery, TestLongPostings, TestNRTReaderWithThreads, TestComplexExplanationsOfNonMatches, TestSparseFixedBitDocIdSet, TestVirtualMethod, TestPrefixFilter, TestGroupFiltering, TestSpanFirstQuery, TestScorerPerf, TestIndexWriterExceptions, Nested, TestRollingBuffer, TestForTooMuchCloning, TestNewestSegment, TestCachingCollector, TestLucene50DocValuesFormat, TestSpans, TestBooleanOr, TestPayloads, TestConstantScoreQuery, ThrowInUncaught, TestMergeSchedulerExternal, TestCharsRef, TestNorms, TestForUtil, TestTransactions, TestFixedBitDocIdSet, TestSumDocFreq, TestTermdocPerf, TestControlledRealTimeReopenThread, TestSimpleFSDirectory, TestWildcardRandom, TestExitableDirectoryReader, TestDemo, TestPositionIncrement, Test4GBStoredFields]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=Test4GBStoredFields -Dtests.seed=A2D310980D63ED69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Pacific/Guadalcanal -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J0 | Test4GBStoredFields (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 14400000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A2D310980D63ED69]:0)
   [junit4] Completed on J0 in 14420.30s, 1 test, 2 errors <<< FAILURES!

[...truncated 30 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 409 suites, 3232 tests, 1 suite-level error, 1 error, 37 ignored (27 assumptions)

Total time: 274 minutes 21 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 3488820 bytes
Compression is 0.0%
Took 1.1 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 726 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/726/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
ERROR: SolrIndexSearcher opens=198 closes=186

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=198 closes=186
	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:447)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:187)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:790)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
12 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=8658, name=searcherExecutor-1977-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=8639, name=searcherExecutor-1975-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=8872, name=searcherExecutor-2110-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=8879, name=searcherExecutor-2137-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=8627, name=searcherExecutor-1969-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=8897, name=searcherExecutor-2140-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=8629, name=searcherExecutor-1970-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=8869, name=searcherExecutor-2115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=8880, name=searcherExecutor-2133-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=8870, name=searcherExecutor-2121-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=8915, name=searcherExecutor-2222-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=8866, name=searcherExecutor-2106-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 12 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
   1) Thread[id=8658, name=searcherExecutor-1977-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=8639, name=searcherExecutor-1975-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=8872, name=searcherExecutor-2110-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=8879, name=searcherExecutor-2137-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=8627, name=searcherExecutor-1969-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=8897, name=searcherExecutor-2140-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=8629, name=searcherExecutor-1970-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=8869, name=searcherExecutor-2115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=8880, name=searcherExecutor-2133-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=8870, name=searcherExecutor-2121-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=8915, name=searcherExecutor-2222-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=8866, name=searcherExecutor-2106-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8866, name=searcherExecutor-2106-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=8915, name=searcherExecutor-2222-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=8870, name=searcherExecutor-2121-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=8629, name=searcherExecutor-1970-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=8869, name=searcherExecutor-2115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=8880, name=searcherExecutor-2133-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=8897, name=searcherExecutor-2140-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=8627, name=searcherExecutor-1969-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=8879, name=searcherExecutor-2137-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=8872, name=searcherExecutor-2110-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=8639, name=searcherExecutor-1975-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=8658, name=searcherExecutor-1977-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8866, name=searcherExecutor-2106-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=8915, name=searcherExecutor-2222-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=8870, name=searcherExecutor-2121-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=8629, name=searcherExecutor-1970-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=8869, name=searcherExecutor-2115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=8880, name=searcherExecutor-2133-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=8897, name=searcherExecutor-2140-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=8627, name=searcherExecutor-1969-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=8879, name=searcherExecutor-2137-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=8872, name=searcherExecutor-2110-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=8639, name=searcherExecutor-1975-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=8658, name=searcherExecutor-1977-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5]:0)


REGRESSION:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5:861B6D9E64691CD9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:480)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:75)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.doTest(LeaderFailoverAfterPartitionTest.java:48)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


REGRESSION:  org.apache.solr.cloud.ReplicationFactorTest.testDistribSearch

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5:861B6D9E64691CD9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:871)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:312)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:226)
	at org.apache.solr.cloud.ReplicationFactorTest.doTest(ReplicationFactorTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:552)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:549)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9981 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/init-core-data-001
   [junit4]   2> 584625 T2303 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /wnl/
   [junit4]   2> 584632 T2303 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 584635 T2303 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 584635 T2304 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 584735 T2303 oasc.ZkTestServer.run start zk server on port:48682
   [junit4]   2> 584736 T2303 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 584738 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 584746 T2311 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e7d1ae0 name:ZooKeeperConnection Watcher:127.0.0.1:48682 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 584746 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 584747 T2303 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 584747 T2303 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 584751 T2303 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 584753 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 584755 T2314 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f7c90af name:ZooKeeperConnection Watcher:127.0.0.1:48682/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 584755 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 584756 T2303 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 584756 T2303 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 584760 T2303 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 584763 T2303 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 584765 T2303 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 584767 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 584768 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 584772 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 584774 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 584877 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 584878 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 584881 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 584882 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 584885 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 584885 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 584887 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 584888 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 584890 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 584891 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 584893 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 584894 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 584896 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 584897 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 584899 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 584900 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 584902 T2303 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 584903 T2303 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 587621 T2303 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 587630 T2303 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48712
   [junit4]   2> 587631 T2303 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 587631 T2303 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 587632 T2303 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002
   [junit4]   2> 587632 T2303 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002/'
   [junit4]   2> 587737 T2303 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002/solr.xml
   [junit4]   2> 587912 T2303 oasc.CoreContainer.<init> New CoreContainer 448692646
   [junit4]   2> 587913 T2303 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002/]
   [junit4]   2> 587915 T2303 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 587916 T2303 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 587917 T2303 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 587917 T2303 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 587918 T2303 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 587918 T2303 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 587919 T2303 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 587919 T2303 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 587920 T2303 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 587920 T2303 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 587922 T2303 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 587922 T2303 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 587923 T2303 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 587923 T2303 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 587924 T2303 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48682/solr
   [junit4]   2> 587925 T2303 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 587925 T2303 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 587926 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 587929 T2326 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@729d30d1 name:ZooKeeperConnection Watcher:127.0.0.1:48682 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 587931 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 587931 T2303 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 587934 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 587937 T2329 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28667cde name:ZooKeeperConnection Watcher:127.0.0.1:48682/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 587937 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 587939 T2303 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 587941 T2303 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 587943 T2303 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 587947 T2303 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 587950 T2303 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 587954 T2303 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 587956 T2303 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48711_wnl
   [junit4]   2> 587957 T2303 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48711_wnl
   [junit4]   2> 587960 T2303 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 587962 T2303 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 587964 T2303 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 587967 T2303 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48711_wnl
   [junit4]   2> 587967 T2303 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 587970 T2303 oasc.Overseer.start Overseer (id=93097749954953219-127.0.0.1:48711_wnl-n_0000000000) starting
   [junit4]   2> 587973 T2303 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 587982 T2303 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 587984 T2331 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 587984 T2303 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 587987 T2303 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 587989 T2303 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 587992 T2330 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 588002 T2333 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 588003 T2333 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 588004 T2329 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 588005 T2333 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 588008 T2330 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48711_wnl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48711/wnl",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 588009 T2330 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48711_wnl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48711/wnl",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 588010 T2330 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 588010 T2330 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 588012 T2329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 589005 T2333 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 589006 T2333 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 589007 T2333 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 589007 T2333 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 589008 T2333 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 589009 T2333 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002/collection1/'
   [junit4]   2> 589012 T2333 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest%207FDE38613367CE5-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 589013 T2333 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest%207FDE38613367CE5-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 589013 T2333 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest%207FDE38613367CE5-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 589074 T2333 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 589079 T2333 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 589105 T2333 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 589149 T2333 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 589201 T2333 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 589303 T2333 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 589315 T2333 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 589381 T3 oasc.CoreContainer.finalize ERROR CoreContainer was not close prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!  instance=1313219283
   [junit4]   2> 589905 T2333 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 589933 T2333 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 589939 T2333 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 589959 T2333 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 589968 T2333 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 589975 T2333 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 589976 T2333 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 589978 T2333 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 589978 T2333 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 589979 T2333 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 589980 T2333 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 589980 T2333 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 589980 T2333 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 589981 T2333 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-001/control/data/
   [junit4]   2> 589981 T2333 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@69fc49ef
   [junit4]   2> 589983 T2333 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-001/control/data
   [junit4]   2> 589983 T2333 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-001/control/data/index/
   [junit4]   2> 589983 T2333 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 589984 T2333 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FDE38613367CE5-001/tempDir-001/control/data/index
   [junit4]   2> 589984 T2333 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=2, maxMergedSegmentMB=85.94921875, floorSegmentMB=1.8447265625, forceMergeDeletesPctAllowed=28.78696450829307, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19853633747525762
   [junit4]   2> 589985 T2333 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@684f9842 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41954a8e),segFN=segments_1,generation=1}
   [junit4]   2> 589986 T2333 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 589992 T2333 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 589992 T2333 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 589992 T2333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 589993 T2333 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 589993 T2333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 589993 T2333 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 589994 T2333 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 589995 T2333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 589995 T2333 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 589996 T2333 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 589997 T2333 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 589997 T2333 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 589997 T2333 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 589998 T2333 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 589999 T2333 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 589999 T2333 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 590000 T2333 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 590000 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 590001 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 590002 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 590003 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 590003 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 590004 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 590004 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 590005 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 590005 T2333 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 590006 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 590006 T2333 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 590069 T2333 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 590075 T2333 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 590080 T2333 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 590084 T2333 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 590087 T2333 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 590095 T2333 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 590096 T2333 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 590097 T2333 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3f3ed966
   [junit4]   2> 590098 T2333 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@684f9842 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41954a8e),segFN=segments_1,generation=1}
   [junit4]   2> 590098 T2333 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 590099 T2333 oass.SolrIndexSearcher.<init> Opening Searcher@40a06220[collection1] main
   [junit4]   2> 590099 T2333 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 590101 T2333 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 590101 T2333 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 590102 T2333 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 590102 T2333 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 590102 T2333 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 590103 T2333 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 590104 T2333 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 590104 T2333 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 590104 T2333 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 590105 T2334 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@40a06220[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 590106 T2333 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 590108 T2337 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48711/wnl collection:control_collection shard:shard1
   [junit4]   2> 590108 T2303 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 590109 T2303 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 590109 T2337 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 590116 T2337 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 590116 T2303 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 590117 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 590118 T2329 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 590119 T2337 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 590119 T2337 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 590120 T2341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@439374eb name:ZooKeeperConnection Watcher:127.0.0.1:48682/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 590120 T2330 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C184 name=collection1 org.apache.solr.core.SolrCore@192c9f0d url=http://127.0.0.1:48711/wnl/collection1 node=127.0.0.1:48711_wnl C184_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:48711_wnl, state=down, base_url=http://127.0.0.1:48711/wnl}
   [junit4]   2> 590120 T2337 C184 P48711 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48711/wnl/collection1/
   [junit4]   2> 590120 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 590121 T2337 C184 P48711 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 590121 T2303 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 590122 T2329 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 590122 T2337 C184 P48711 oasc.SyncStrategy.syncToMe http://127.0.0.1:48711/wnl/collection1/ has no replicas
   [junit4]   2> 590123 T2337 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48711/wnl/collection1/ shard1
   [junit4]   2> 590123 T2303 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 590123 T2337 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 590127 T2303 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 590128 T2303 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 590128 T2303 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 590129 T2342 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 590129 T2303 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 590131 T2345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@109a5929 name:ZooKeeperConnection Watcher:127.0.0.1:48682/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 590132 T2330 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48711/wnl",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 590132 T2303 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 590133 T2303 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 590135 T2342 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 590136 T2330 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":2} current state version: 2
   [junit4]   2> 590137 T2330 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 590137 T2341 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 590242 T2330 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 590245 T2341 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 590283 T2337 oasc.ZkController.register We are http://127.0.0.1:48711/wnl/collection1/ and leader is http://127.0.0.1:48711/wnl/collection1/
   [junit4]   2> 590283 T2337 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48711/wnl
   [junit4]   2> 590283 T2337 oasc.ZkControl

[...truncated too long message...]

afe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=8869, name=searcherExecutor-2115-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    6) Thread[id=8880, name=searcherExecutor-2133-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=8897, name=searcherExecutor-2140-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    8) Thread[id=8627, name=searcherExecutor-1969-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    9) Thread[id=8879, name=searcherExecutor-2137-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=8872, name=searcherExecutor-2110-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   11) Thread[id=8639, name=searcherExecutor-1975-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   12) Thread[id=8658, name=searcherExecutor-1977-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FDE38613367CE5]:0)
   [junit4] Completed on J3 in 273.21s, 1 test, 1 failure, 2 errors <<< FAILURES!

[...truncated 703 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 460 suites, 1870 tests, 3 suite-level errors, 2 errors, 814 ignored (8 assumptions)

Total time: 137 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 602571926 bytes
Compression is 0.0%
Took 1 min 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 725 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/725/

2 tests failed.
REGRESSION:  org.apache.solr.core.TestDynamicLoading.testDistribSearch

Error Message:
New version of class is not loaded {   "responseHeader":{     "status":404,     "QTime":4},   "error":{     "msg":"no such blob or version available: test/2",     "code":404}}

Stack Trace:
java.lang.AssertionError: New version of class is not loaded {
  "responseHeader":{
    "status":404,
    "QTime":4},
  "error":{
    "msg":"no such blob or version available: test/2",
    "code":404}}
	at __randomizedtesting.SeedInfo.seed([505DE86E4518925D:D1BB66763247F261]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestDynamicLoading.dynamicLoading(TestDynamicLoading.java:154)
	at org.apache.solr.core.TestDynamicLoading.doTest(TestDynamicLoading.java:64)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:22278: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:22278: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:449)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:201)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10301 lines...]
   [junit4] Suite: org.apache.solr.core.TestDynamicLoading
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/init-core-data-001
   [junit4]   2> 1746802 T3645 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1746802 T3645 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pkflp/
   [junit4]   2> 1746807 T3645 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1746808 T3645 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1746809 T3646 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1746909 T3645 oasc.ZkTestServer.run start zk server on port:32440
   [junit4]   2> 1746910 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1746911 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1746916 T3653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19e28a07 name:ZooKeeperConnection Watcher:127.0.0.1:32440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1746917 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1746917 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1746917 T3645 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1746921 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1746922 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1746923 T3656 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@564315ff name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1746923 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1746924 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1746924 T3645 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1746926 T3645 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1746928 T3645 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1746929 T3645 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1746931 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1746932 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1746935 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1746935 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1747038 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1747039 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1747041 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1747041 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1747043 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1747044 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1747046 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1747046 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1747048 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1747049 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1747051 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1747051 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1747053 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1747054 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1747056 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1747056 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1747058 T3645 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1747059 T3645 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1748318 T3645 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1748322 T3645 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54825
   [junit4]   2> 1748322 T3645 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1748322 T3645 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1748323 T3645 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002
   [junit4]   2> 1748323 T3645 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/'
   [junit4]   2> 1748354 T3645 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/solr.xml
   [junit4]   2> 1748423 T3645 oasc.CoreContainer.<init> New CoreContainer 681422295
   [junit4]   2> 1748423 T3645 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/]
   [junit4]   2> 1748424 T3645 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1748425 T3645 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1748425 T3645 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1748425 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1748425 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1748425 T3645 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1748426 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1748426 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1748426 T3645 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1748426 T3645 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1748427 T3645 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1748427 T3645 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1748427 T3645 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1748427 T3645 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1748428 T3645 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32440/solr
   [junit4]   2> 1748428 T3645 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1748428 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1748429 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1748431 T3668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63e467b9 name:ZooKeeperConnection Watcher:127.0.0.1:32440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1748432 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1748432 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1748434 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1748435 T3671 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@764fdedb name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1748436 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1748437 T3645 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1748438 T3645 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1748440 T3645 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1748441 T3645 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1748442 T3645 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1748444 T3645 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1748445 T3645 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54825_pkflp
   [junit4]   2> 1748445 T3645 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54825_pkflp
   [junit4]   2> 1748447 T3645 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1748448 T3645 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1748449 T3645 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1748450 T3645 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54825_pkflp
   [junit4]   2> 1748450 T3645 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1748451 T3645 oasc.Overseer.start Overseer (id=93092307960856579-127.0.0.1:54825_pkflp-n_0000000000) starting
   [junit4]   2> 1748452 T3645 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1748456 T3645 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1748457 T3673 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1748457 T3645 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1748459 T3645 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1748460 T3645 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1748461 T3672 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1748465 T3675 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1748466 T3675 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1748466 T3675 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1748466 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1748468 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54825/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54825_pkflp"} current state version: 0
   [junit4]   2> 1748469 T3672 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54825/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54825_pkflp"}
   [junit4]   2> 1748469 T3672 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1748469 T3672 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1748470 T3671 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1749466 T3675 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1749467 T3675 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1749467 T3675 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1749468 T3675 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1749468 T3675 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1749469 T3675 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/collection1/'
   [junit4]   2> 1749471 T3675 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1749472 T3675 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1749472 T3675 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1749506 T3675 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1749509 T3675 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1749521 T3675 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1749545 T3675 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 1749608 T3675 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1749709 T3675 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1749719 T3675 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1750094 T3675 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1750107 T3675 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1750109 T3675 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1750119 T3675 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1750123 T3675 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1750127 T3675 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1750128 T3675 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1750128 T3675 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1750128 T3675 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1750129 T3675 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1750129 T3675 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1750130 T3675 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1750130 T3675 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1750130 T3675 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/control/data/
   [junit4]   2> 1750130 T3675 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c2b74c4
   [junit4]   2> 1750132 T3675 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/control/data
   [junit4]   2> 1750132 T3675 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/control/data/index/
   [junit4]   2> 1750132 T3675 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1750132 T3675 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/control/data/index
   [junit4]   2> 1750133 T3675 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=90.9248046875, floorSegmentMB=1.0673828125, forceMergeDeletesPctAllowed=20.268804703871318, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1750134 T3675 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2858cb0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d7b5a7a),segFN=segments_1,generation=1}
   [junit4]   2> 1750134 T3675 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1750138 T3675 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1750139 T3675 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1750139 T3675 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1750139 T3675 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1750139 T3675 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1750140 T3675 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1750140 T3675 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1750140 T3675 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1750141 T3675 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1750141 T3675 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1750142 T3675 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1750143 T3675 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1750143 T3675 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1750144 T3675 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1750145 T3675 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1750145 T3675 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1750146 T3675 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1750147 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1750149 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1750150 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1750150 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1750151 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1750152 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1750152 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1750153 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1750154 T3675 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1750154 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1750155 T3675 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1750174 T3675 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1750176 T3675 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1750178 T3675 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1750180 T3675 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1750182 T3675 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1750186 T3675 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1750186 T3675 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1750187 T3675 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=78.98828125, floorSegmentMB=1.31640625, forceMergeDeletesPctAllowed=18.400213710672894, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1750187 T3675 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2858cb0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d7b5a7a),segFN=segments_1,generation=1}
   [junit4]   2> 1750188 T3675 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1750188 T3675 oass.SolrIndexSearcher.<init> Opening Searcher@385aab6c[collection1] main
   [junit4]   2> 1750188 T3675 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1750189 T3675 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1750189 T3675 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1750190 T3675 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1750190 T3675 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1750190 T3675 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1750191 T3675 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1750191 T3675 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1750191 T3675 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1750192 T3675 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1750192 T3675 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1750193 T3676 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@385aab6c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1750193 T3679 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54825/pkflp collection:control_collection shard:shard1
   [junit4]   2> 1750193 T3645 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 1750194 T3645 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1750194 T3679 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1750194 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1750195 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1750197 T3682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@185662ac name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1750197 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1750198 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1750199 T3645 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1750200 T3679 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1750201 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1750202 T3679 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1750202 T3679 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1750202 T3645 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1750202 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C7837 name=collection1 org.apache.solr.core.SolrCore@e80d0a6 url=http://127.0.0.1:54825/pkflp/collection1 node=127.0.0.1:54825_pkflp C7837_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:54825/pkflp, core=collection1, state=down, node_name=127.0.0.1:54825_pkflp}
   [junit4]   2> 1750202 T3679 C7837 P54825 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54825/pkflp/collection1/
   [junit4]   2> 1750203 T3645 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1750203 T3679 C7837 P54825 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1750203 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1750204 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1750204 T3671 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1750204 T3679 C7837 P54825 oasc.SyncStrategy.syncToMe http://127.0.0.1:54825/pkflp/collection1/ has no replicas
   [junit4]   2> 1750205 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1750205 T3679 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54825/pkflp/collection1/ shard1
   [junit4]   2> 1750206 T3679 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1750206 T3685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c084bb9 name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1750207 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1750207 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1750209 T3686 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1750211 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 2
   [junit4]   2> 1750211 T3672 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1750214 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54825/pkflp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1750214 T3672 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1750215 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1750218 T3686 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1750219 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1750261 T3679 oasc.ZkController.register We are http://127.0.0.1:54825/pkflp/collection1/ and leader is http://127.0.0.1:54825/pkflp/collection1/
   [junit4]   2> 1750262 T3679 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54825/pkflp
   [junit4]   2> 1750262 T3679 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1750262 T3679 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1750262 T3679 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1750263 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1750265 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54825/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54825_pkflp"} current state version: 4
   [junit4]   2> 1750265 T3672 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54825/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54825_pkflp"}
   [junit4]   2> 1750368 T3671 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1750368 T3682 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1751382 T3645 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1751383 T3645 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1751386 T3645 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50889
   [junit4]   2> 1751386 T3645 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1751386 T3645 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1751387 T3645 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003
   [junit4]   2> 1751387 T3645 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/'
   [junit4]   2> 1751418 T3645 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/solr.xml
   [junit4]   2> 1751471 T3645 oasc.CoreContainer.<init> New CoreContainer 280510294
   [junit4]   2> 1751472 T3645 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/]
   [junit4]   2> 1751473 T3645 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1751473 T3645 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1751473 T3645 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1751473 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1751474 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1751474 T3645 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1751474 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1751474 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1751474 T3645 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1751475 T3645 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1751475 T3645 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1751475 T3645 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1751476 T3645 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1751476 T3645 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1751476 T3645 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32440/solr
   [junit4]   2> 1751476 T3645 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1751477 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1751477 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1751479 T3698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40ce26e8 name:ZooKeeperConnection Watcher:127.0.0.1:32440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1751480 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1751480 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1751482 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1751483 T3701 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45a470dc name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1751483 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1751486 T3645 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1752490 T3645 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50889_pkflp
   [junit4]   2> 1752491 T3645 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50889_pkflp
   [junit4]   2> 1752494 T3645 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1752502 T3702 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1752503 T3702 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1752504 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1752504 T3702 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1752505 T3702 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1752505 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50889/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:50889_pkflp"} current state version: 5
   [junit4]   2> 1752505 T3702 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1752505 T3672 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50889/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:50889_pkflp"}
   [junit4]   2> 1752506 T3672 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 1752506 T3672 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1752506 T3702 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1752506 T3702 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1752507 T3672 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1752508 T3701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1752508 T3701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1753507 T3702 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1753508 T3702 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1753508 T3702 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1753509 T3702 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1753509 T3702 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1753509 T3702 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/collection1/'
   [junit4]   2> 1753512 T3702 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1753513 T3702 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1753513 T3702 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1753549 T3702 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1753552 T3702 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1753564 T3702 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1753587 T3702 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 1753636 T3702 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1753737 T3702 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1753748 T3702 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1754138 T3702 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1754151 T3702 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1754154 T3702 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1754164 T3702 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1754168 T3702 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1754172 T3702 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1754173 T3702 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1754174 T3702 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1754174 T3702 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1754175 T3702 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1754175 T3702 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1754176 T3702 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1754176 T3702 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1754176 T3702 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty1/
   [junit4]   2> 1754176 T3702 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c2b74c4
   [junit4]   2> 1754177 T3702 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty1
   [junit4]   2> 1754178 T3702 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty1/index/
   [junit4]   2> 1754178 T3702 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1754178 T3702 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty1/index
   [junit4]   2> 1754179 T3702 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=90.9248046875, floorSegmentMB=1.0673828125, forceMergeDeletesPctAllowed=20.268804703871318, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1754179 T3702 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e31cbf6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@574592f6),segFN=segments_1,generation=1}
   [junit4]   2> 1754180 T3702 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1754185 T3702 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1754185 T3702 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1754185 T3702 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1754186 T3702 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1754186 T3702 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1754186 T3702 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1754186 T3702 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1754187 T3702 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1754187 T3702 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1754188 T3702 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1754189 T3702 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1754190 T3702 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1754191 T3702 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 1754192 T3702 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 1754193 T3702 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 1754194 T3702 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 1754195 T3702 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 1754196 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 1754198 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 1754199 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 1754200 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 1754202 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 1754203 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 1754204 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 1754205 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 1754206 T3702 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1754208 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1754210 T3702 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 1754229 T3702 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1754231 T3702 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1754233 T3702 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1754235 T3702 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1754237 T3702 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1754239 T3702 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1754240 T3702 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1754240 T3702 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=78.98828125, floorSegmentMB=1.31640625, forceMergeDeletesPctAllowed=18.400213710672894, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1754241 T3702 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e31cbf6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@574592f6),segFN=segments_1,generation=1}
   [junit4]   2> 1754241 T3702 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1754242 T3702 oass.SolrIndexSearcher.<init> Opening Searcher@3c694d6e[collection1] main
   [junit4]   2> 1754242 T3702 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1754243 T3702 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1754243 T3702 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1754243 T3702 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1754244 T3702 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1754244 T3702 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1754244 T3702 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1754245 T3702 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1754245 T3702 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1754245 T3702 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1754246 T3703 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3c694d6e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1754247 T3702 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1754247 T3706 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50889/pkflp collection:collection1 shard:shard2
   [junit4]   2> 1754248 T3645 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 1754248 T3645 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1754248 T3706 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1754253 T3706 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1754254 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1754254 T3706 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1754255 T3706 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C7838 name=collection1 org.apache.solr.core.SolrCore@7d214d17 url=http://127.0.0.1:50889/pkflp/collection1 node=127.0.0.1:50889_pkflp C7838_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:50889/pkflp, core=collection1, state=down, node_name=127.0.0.1:50889_pkflp}
   [junit4]   2> 1754255 T3706 C7838 P50889 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50889/pkflp/collection1/
   [junit4]   2> 1754255 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1754255 T3706 C7838 P50889 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1754255 T3706 C7838 P50889 oasc.SyncStrategy.syncToMe http://127.0.0.1:50889/pkflp/collection1/ has no replicas
   [junit4]   2> 1754256 T3706 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50889/pkflp/collection1/ shard2
   [junit4]   2> 1754256 T3706 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1754256 T3672 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1754257 T3701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1754258 T3701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1754260 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50889/pkflp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1754261 T3672 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1754262 T3701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1754262 T3701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1754264 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1754310 T3706 oasc.ZkController.register We are http://127.0.0.1:50889/pkflp/collection1/ and leader is http://127.0.0.1:50889/pkflp/collection1/
   [junit4]   2> 1754311 T3706 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50889/pkflp
   [junit4]   2> 1754311 T3706 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1754311 T3706 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1754311 T3706 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1754313 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1754314 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50889/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:50889_pkflp"} current state version: 5
   [junit4]   2> 1754314 T3672 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50889/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:50889_pkflp"}
   [junit4]   2> 1754315 T3706 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1754315 T3672 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1754316 T3701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1754317 T3701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1755422 T3645 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1755422 T3645 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1755425 T3645 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55605
   [junit4]   2> 1755425 T3645 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1755425 T3645 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1755426 T3645 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004
   [junit4]   2> 1755426 T3645 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/'
   [junit4]   2> 1755456 T3645 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/solr.xml
   [junit4]   2> 1755532 T3645 oasc.CoreContainer.<init> New CoreContainer 1541588621
   [junit4]   2> 1755533 T3645 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/]
   [junit4]   2> 1755534 T3645 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1755535 T3645 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1755535 T3645 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1755535 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1755536 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1755536 T3645 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1755536 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1755537 T3645 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1755537 T3645 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1755537 T3645 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1755538 T3645 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 1755538 T3645 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1755539 T3645 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1755539 T3645 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1755539 T3645 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32440/solr
   [junit4]   2> 1755540 T3645 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1755540 T3645 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1755541 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1755543 T3718 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64e4a43e name:ZooKeeperConnection Watcher:127.0.0.1:32440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1755544 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1755544 T3645 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1755546 T3645 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1755547 T3721 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b03700d name:ZooKeeperConnection Watcher:127.0.0.1:32440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1755547 T3645 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1755551 T3645 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1756555 T3645 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55605_pkflp
   [junit4]   2> 1756556 T3645 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55605_pkflp
   [junit4]   2> 1756559 T3645 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1756566 T3722 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1756566 T3722 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1756567 T3671 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1756568 T3722 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1756568 T3722 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1756568 T3672 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55605/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55605_pkflp"} current state version: 5
   [junit4]   2> 1756568 T3722 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1756569 T3672 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55605/pkflp",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55605_pkflp"}
   [junit4]   2> 1756569 T3672 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 1756569 T3672 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1756569 T3722 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1756569 T3722 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1756570 T3672 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 1756570 T3701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1756570 T3721 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 1756571 T3701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1756571 T3721 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1757570 T3722 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1757571 T3722 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1757571 T3722 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1757572 T3722 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1757572 T3722 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1757572 T3722 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/collection1/'
   [junit4]   2> 1757574 T3722 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1757575 T3722 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1757575 T3722 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1757610 T3722 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1757612 T3722 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1757624 T3722 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1757647 T3722 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 1757695 T3722 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1757796 T3722 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1757806 T3722 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1758213 T3722 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1758225 T3722 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1758228 T3722 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1758237 T3722 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1758242 T3722 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1758245 T3722 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1758246 T3722 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1758247 T3722 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1758247 T3722 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1758248 T3722 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1758248 T3722 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1758249 T3722 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1758249 T3722 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1758249 T3722 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-004/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty2/
   [junit4]   2> 1758249 T3722 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c2b74c4
   [junit4]   2> 1758251 T3722 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty2
   [junit4]   2> 1758251 T3722 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty2/index/
   [junit4]   2> 1758251 T3722 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1758251 T3722 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.core.TestDynamicLoading-505DE86E4518925D-001/tempDir-001/jetty2/index
   [junit4]   2> 1758252 T3722 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=90.9248046875, floorSegmentMB=1.0673828125, forceMergeDeletesPctAllowed=20.268804703871318, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1758253 T3722 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e1c1caa lockFactory=org.apache.

[...truncated too long message...]

.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4641, name=Thread-2078, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:22268, http://127.0.0.1:22287, http://127.0.0.1:22314, http://127.0.0.1:22278, http://127.0.0.1:22297]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([505DE86E4518925D]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:22268, http://127.0.0.1:22287, http://127.0.0.1:22314, http://127.0.0.1:22278, http://127.0.0.1:22297]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:22297: Could not fully create collection: awholynewstresscollection_collection2_1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4639, name=Thread-2076, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:22268, http://127.0.0.1:22287, http://127.0.0.1:22314, http://127.0.0.1:22278, http://127.0.0.1:22297]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([505DE86E4518925D]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:22268, http://127.0.0.1:22287, http://127.0.0.1:22314, http://127.0.0.1:22278, http://127.0.0.1:22297]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:22297: Could not fully create collection: awholynewstresscollection_collection0_1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 more
   [junit4]   2> 1872247 T4115 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.CollectionsAPIDistributedZkTest-505DE86E4518925D-001
   [junit4]   2> 914379 T4114 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=FSTOrd50, range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), a_i1=PostingsFormat(name=LuceneVarGapDocFreqInterval), other_tl1=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=pl_PL, timezone=Asia/Bahrain
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=68252712,total=449314816
   [junit4]   2> NOTE: All tests run in this JVM: [FastVectorHighlighterTest, TestCodecSupport, TestClusterStateMutator, DistributedSpellCheckComponentTest, TestRandomFaceting, SimpleFacetsTest, TestStressReorder, NumericFieldsTest, TestDFRSimilarityFactory, ShardRoutingCustomTest, TestCopyFieldCollectionResource, TimeZoneUtilsTest, TestPartialUpdateDeduplication, OverseerRolesTest, ZkSolrClientTest, PolyFieldTest, TestFreeTextSuggestions, DistribDocExpirationUpdateProcessorTest, SolrCmdDistributorTest, TestInfoStreamLogging, ShowFileRequestHandlerTest, DistributedExpandComponentTest, TestIBSimilarityFactory, TestRTGBase, TestSchemaVersionResource, OverseerCollectionProcessorTest, AddSchemaFieldsUpdateProcessorFactoryTest, OverriddenZkACLAndCredentialsProvidersTest, TestTolerantSearch, InfoHandlerTest, CursorPagingTest, CursorMarkTest, ExitableDirectoryReaderTest, RemoteQueryErrorTest, TestModifyConfFiles, TestCoreContainer, TestAtomicUpdateErrorCases, SignatureUpdateProcessorFactoryTest, BlockDirectoryTest, TestMergePolicyConfig, OverseerStatusTest, BadCopyFieldTest, SampleTest, TestUtils, PrimitiveFieldTypeTest, CopyFieldTest, TestUpdate, TestFuzzyAnalyzedSuggestions, StatelessScriptUpdateProcessorFactoryTest, TestFastLRUCache, SuggesterTest, TestCloudManagedSchema, TestCloudPivotFacet, TestDistributedMissingSort, TestFaceting, RegexBoostProcessorTest, TestSolrXmlPersistor, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J0 in 914.68s, 1 test, 1 error <<< FAILURES!

[...truncated 742 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 460 suites, 1869 tests, 1 error, 1 failure, 57 ignored (10 assumptions)

Total time: 210 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 939988058 bytes
Compression is 0.0%
Took 2 min 52 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 724 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/724/

1 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:51063/_uu/uj: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51063/_uu/uj: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:449)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:201)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10626 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/init-core-data-001
   [junit4]   2> 984108 T21330 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 984108 T21330 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_uu/uj
   [junit4]   2> 984114 T21330 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 984115 T21330 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 984115 T21331 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 984215 T21330 oasc.ZkTestServer.run start zk server on port:51052
   [junit4]   2> 984216 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 984217 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 984221 T21338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a6ea9d8 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 984222 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 984222 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 984222 T21330 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 984226 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 984226 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 984228 T21341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29bbcdca name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 984228 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 984229 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 984229 T21330 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 984232 T21330 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 984234 T21330 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 984236 T21330 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 984238 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 984239 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 984243 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 984243 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 984346 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 984347 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 984349 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 984350 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 984352 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 984353 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 984355 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 984356 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 984358 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 984359 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 984361 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 984361 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 984364 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 984364 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 984366 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 984367 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 984369 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 984370 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using old style solr.xml
   [junit4]   2> 984373 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 984374 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 984376 T21344 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@479d58b0 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 984376 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 984376 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 984377 T21330 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 984379 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 984380 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 984381 T21347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@777291d6 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 984381 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 984382 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 984383 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 984384 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 984387 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 984388 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 984490 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 984491 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 984493 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 984493 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 984495 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 984496 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 984498 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 984499 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 984501 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 984501 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 984503 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 984504 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 984506 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 984506 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 984508 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 984509 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 984511 T21330 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 984512 T21330 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 985929 T21330 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 985934 T21330 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51057
   [junit4]   2> 985935 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 985935 T21330 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 985936 T21330 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002
   [junit4]   2> 985936 T21330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/'
   [junit4]   2> 985979 T21330 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/solr.xml
   [junit4]   2> 986054 T21330 oasc.CoreContainer.<init> New CoreContainer 1901894228
   [junit4]   2> 986055 T21330 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/]
   [junit4]   2> 986056 T21330 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 986056 T21330 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 986057 T21330 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 986057 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 986057 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 986058 T21330 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 986058 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 986058 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 986059 T21330 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 986059 T21330 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 986060 T21330 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 986060 T21330 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 986060 T21330 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 986061 T21330 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 986061 T21330 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51052/solr
   [junit4]   2> 986061 T21330 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 986062 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 986062 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 986064 T21359 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d4ef803 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 986065 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 986065 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 986067 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 986069 T21362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7489b582 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 986069 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 986071 T21330 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 986073 T21330 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 986075 T21330 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 986077 T21330 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 986080 T21330 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 986082 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 986083 T21330 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51057__uu%2Fuj
   [junit4]   2> 986084 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51057__uu%2Fuj
   [junit4]   2> 986086 T21330 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 986088 T21330 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 986089 T21330 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 986091 T21330 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51057__uu%2Fuj
   [junit4]   2> 986091 T21330 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 986093 T21330 oasc.Overseer.start Overseer (id=93088618016079877-127.0.0.1:51057__uu%2Fuj-n_0000000000) starting
   [junit4]   2> 986095 T21330 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 986102 T21330 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 986102 T21364 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 986103 T21330 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 986105 T21330 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 986106 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 986109 T21363 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 986113 T21330 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 986114 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 986114 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 986115 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 986117 T21369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6adc80ad name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 986117 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 986117 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 986118 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 986121 T21330 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 986121 T21330 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 986122 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 986122 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 986124 T21372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77bec2f1 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 986124 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 986124 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 986127 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 986130 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 0
   [junit4]   2> 986130 T21363 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 986132 T21363 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 986133 T21362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 986133 T21369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 987704 T21330 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 987705 T21330 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 987708 T21330 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51063
   [junit4]   2> 987709 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 987709 T21330 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 987710 T21330 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-003
   [junit4]   2> 987710 T21330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-003/'
   [junit4]   2> 987765 T21330 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-003/solr.xml
   [junit4]   2> 987854 T21330 oasc.CoreContainer.<init> New CoreContainer 1555405928
   [junit4]   2> 987854 T21330 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-003/]
   [junit4]   2> 987856 T21330 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 987856 T21330 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 987856 T21330 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 987857 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 987857 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 987857 T21330 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 987858 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 987858 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 987858 T21330 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 987859 T21330 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 987859 T21330 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 987860 T21330 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 987860 T21330 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 987860 T21330 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 987861 T21330 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51052/solr
   [junit4]   2> 987861 T21330 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 987862 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 987863 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 987864 T21384 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d30815c name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 987865 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 987865 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 987868 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 987869 T21387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20477028 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 987870 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 987878 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 988884 T21330 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51063__uu%2Fuj
   [junit4]   2> 988886 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51063__uu%2Fuj
   [junit4]   2> 988890 T21330 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 988896 T21330 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 988896 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 990369 T21330 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 990370 T21330 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 990373 T21330 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51069
   [junit4]   2> 990374 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 990374 T21330 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 990374 T21330 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-004
   [junit4]   2> 990375 T21330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-004/'
   [junit4]   2> 990424 T21330 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-004/solr.xml
   [junit4]   2> 990510 T21330 oasc.CoreContainer.<init> New CoreContainer 829960066
   [junit4]   2> 990510 T21330 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-004/]
   [junit4]   2> 990511 T21330 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 990512 T21330 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 990512 T21330 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 990512 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 990513 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 990513 T21330 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 990513 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 990514 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 990514 T21330 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 990515 T21330 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 990515 T21330 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 990516 T21330 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 990516 T21330 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 990516 T21330 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 990517 T21330 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51052/solr
   [junit4]   2> 990517 T21330 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 990518 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 990518 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 990520 T21400 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45b62301 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 990521 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 990521 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 990524 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 990525 T21403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2591bc68 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 990526 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 990532 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 991537 T21330 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51069__uu%2Fuj
   [junit4]   2> 991538 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51069__uu%2Fuj
   [junit4]   2> 991542 T21330 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 991547 T21330 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 991548 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 992943 T21330 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 992944 T21330 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 992947 T21330 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51075
   [junit4]   2> 992948 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 992949 T21330 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 992949 T21330 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005
   [junit4]   2> 992949 T21330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/'
   [junit4]   2> 993002 T21330 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/solr.xml
   [junit4]   2> 993084 T21330 oasc.CoreContainer.<init> New CoreContainer 2080973083
   [junit4]   2> 993085 T21330 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/]
   [junit4]   2> 993086 T21330 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 993087 T21330 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 993087 T21330 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 993087 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 993088 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 993088 T21330 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 993088 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 993089 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 993089 T21330 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 993089 T21330 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 993090 T21330 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 993090 T21330 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 993091 T21330 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 993091 T21330 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 993092 T21330 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51052/solr
   [junit4]   2> 993092 T21330 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 993092 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 993093 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 993095 T21416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@43f71424 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 993095 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 993096 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 993098 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 993100 T21419 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c1793bb name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 993100 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 993106 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 994112 T21330 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51075__uu%2Fuj
   [junit4]   2> 994113 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51075__uu%2Fuj
   [junit4]   2> 994117 T21330 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 994122 T21330 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 994123 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 995558 T21330 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 995559 T21330 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 995562 T21330 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51088
   [junit4]   2> 995563 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 995563 T21330 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 995564 T21330 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-006
   [junit4]   2> 995564 T21330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-006/'
   [junit4]   2> 995610 T21330 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-006/solr.xml
   [junit4]   2> 995690 T21330 oasc.CoreContainer.<init> New CoreContainer 253315436
   [junit4]   2> 995691 T21330 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-006/]
   [junit4]   2> 995692 T21330 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 995692 T21330 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 995693 T21330 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 995693 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 995693 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 995694 T21330 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 995694 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 995694 T21330 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 995695 T21330 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 995695 T21330 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 995696 T21330 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 995696 T21330 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 995696 T21330 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 995697 T21330 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 995697 T21330 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51052/solr
   [junit4]   2> 995697 T21330 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 995698 T21330 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 995699 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 995701 T21432 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bae1093 name:ZooKeeperConnection Watcher:127.0.0.1:51052 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 995701 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 995701 T21330 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 995704 T21330 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 995707 T21435 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@331094a2 name:ZooKeeperConnection Watcher:127.0.0.1:51052/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 995707 T21330 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 995713 T21330 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 996718 T21330 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51088__uu%2Fuj
   [junit4]   2> 996719 T21330 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51088__uu%2Fuj
   [junit4]   2> 996723 T21330 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 996729 T21330 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 996729 T21330 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 996733 T21376 oasha.CollectionsHandler.handleCreateAction Creating Collection : version=2&action=CREATE&collection.configName=conf1&name=nodes_used_collection&numShards=2&wt=javabin&replicationFactor=2
   [junit4]   2> 996734 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/collection-queue-work state SyncConnected
   [junit4]   2> 996736 T21364 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 996737 T21437 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 996738 T21437 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/nodes_used_collection 
   [junit4]   2> 996738 T21437 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 996741 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 996743 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2"} current state version: 1
   [junit4]   2> 996743 T21363 oasco.ClusterStateMutator.createCollection building a new cName: nodes_used_collection
   [junit4]   2> 996745 T21387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996745 T21419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996745 T21369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996745 T21435 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996745 T21362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996745 T21403 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996943 T21437 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection nodes_used_collection, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 996944 T21437 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:51075__uu%2Fuj
   [junit4]   2> 996944 T21437 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:51069__uu%2Fuj
   [junit4]   2> 996946 T21437 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:51088__uu%2Fuj
   [junit4]   2> 996947 T21437 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:51057__uu%2Fuj
   [junit4]   2> 996947 T21411 oasha.CoreAdminHandler.handleCreateAction core create command version=2&action=CREATE&collection=nodes_used_collection&collection.configName=conf1&name=nodes_used_collection_shard1_replica1&numShards=2&shard=shard1&wt=javabin&qt=/admin/cores
   [junit4]   2> 996947 T21396 oasha.CoreAdminHandler.handleCreateAction core create command version=2&action=CREATE&collection=nodes_used_collection&collection.configName=conf1&name=nodes_used_collection_shard1_replica2&numShards=2&shard=shard1&wt=javabin&qt=/admin/cores
   [junit4]   2> 996948 T21425 oasha.CoreAdminHandler.handleCreateAction core create command version=2&action=CREATE&collection=nodes_used_collection&collection.configName=conf1&name=nodes_used_collection_shard2_replica1&numShards=2&shard=shard2&wt=javabin&qt=/admin/cores
   [junit4]   2> 996950 T21351 oasha.CoreAdminHandler.handleCreateAction core create command version=2&action=CREATE&collection=nodes_used_collection&collection.configName=conf1&name=nodes_used_collection_shard2_replica2&numShards=2&shard=shard2&wt=javabin&qt=/admin/cores
   [junit4]   2> 996950 T21411 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 996951 T21396 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 996951 T21411 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 996952 T21425 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 996951 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 996952 T21396 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 996953 T21425 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 996953 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51075__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51075/_uu/uj"} current state version: 2
   [junit4]   2> 996953 T21351 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 996953 T21363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51075__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51075/_uu/uj"}
   [junit4]   2> 996954 T21351 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 996956 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51069__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51069/_uu/uj"} current state version: 2
   [junit4]   2> 996956 T21363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51069__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51069/_uu/uj"}
   [junit4]   2> 996958 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 996958 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51088__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51088/_uu/uj"} current state version: 2
   [junit4]   2> 996959 T21363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51088__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51088/_uu/uj"}
   [junit4]   2> 996961 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51057__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51057/_uu/uj"} current state version: 2
   [junit4]   2> 996961 T21363 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:51057__uu%2Fuj",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51057/_uu/uj"}
   [junit4]   2> 996963 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 996964 T21369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996964 T21387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996964 T21419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996964 T21403 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996964 T21362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 996964 T21435 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 997952 T21396 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 997953 T21411 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 997953 T21396 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 997953 T21425 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 997953 T21411 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 997954 T21425 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 997955 T21411 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 997955 T21425 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 997955 T21351 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 997954 T21396 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 997956 T21351 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 997955 T21425 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 997957 T21351 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 997955 T21411 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 997957 T21425 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 997957 T21351 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 997956 T21396 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 997958 T21411 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 997958 T21425 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-006/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 997959 T21411 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 997959 T21351 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 997959 T21396 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 997960 T21351 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 997960 T21396 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-004/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 997981 T21411 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 997985 T21411 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 997988 T21396 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 997991 T21396 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 997993 T21425 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 997995 T21425 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 997998 T21351 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 998000 T21411 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 998002 T21351 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 998002 T21396 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 998005 T21425 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 998013 T21351 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 998029 T21411 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 998032 T21396 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 998035 T21425 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 998053 T21351 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 998101 T21411 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 998105 T21396 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 998106 T21425 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 998112 T21351 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 998113 T21351 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 998125 T21351 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 998203 T21411 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 998206 T21396 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 998207 T21425 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 998217 T21411 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 998219 T21425 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 998219 T21396 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 998598 T21351 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 998612 T21351 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 998616 T21351 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 998629 T21351 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998634 T21351 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998640 T21351 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 998641 T21351 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 998641 T21351 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 998642 T21351 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 998643 T21351 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 998643 T21351 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 998644 T21351 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using configuration from collection nodes_used_collection
   [junit4]   2> 998644 T21351 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 998645 T21351 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 998646 T21351 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f7b130c
   [junit4]   2> 998647 T21351 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/data
   [junit4]   2> 998648 T21351 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 998648 T21351 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 998649 T21351 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-002/nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 998650 T21351 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=1.8212890625, floorSegmentMB=0.7001953125, forceMergeDeletesPctAllowed=28.589525972276046, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4343149677916873
   [junit4]   2> 998651 T21351 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@233a494f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47a2f39a),segFN=segments_1,generation=1}
   [junit4]   2> 998652 T21351 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 998660 T21351 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 998661 T21351 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 998661 T21351 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 998662 T21351 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 998662 T21351 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 998663 T21351 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 998664 T21351 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 998664 T21351 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 998664 T21351 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 998665 T21351 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 998666 T21351 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 998667 T21351 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 998667 T21351 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 998668 T21351 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 998669 T21351 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 998669 T21351 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 998670 T21351 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 998671 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 998672 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 998673 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 998674 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 998675 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 998676 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 998677 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 998678 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 998679 T21351 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 998680 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 998681 T21351 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 998720 T21351 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 998723 T21351 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 998724 T21411 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 998737 T21351 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 998740 T21351 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 998742 T21411 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 998743 T21351 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 998745 T21396 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 998746 T21411 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 998748 T21351 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 998748 T21351 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 998749 T21351 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5a464ac1
   [junit4]   2> 998750 T21351 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@233a494f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@47a2f39a),segFN=segments_1,generation=1}
   [junit4]   2> 998750 T21351 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 998751 T21351 oass.SolrIndexSearcher.<init> Opening Searcher@58e5d983[nodes_used_collection_shard2_replica2] main
   [junit4]   2> 998751 T21351 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 998752 T21351 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 998753 T21351 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 998752 T21396 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 998753 T21351 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 998754 T21351 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 998754 T21351 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 998755 T21351 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 998756 T21351 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 998756 T21425 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 998756 T21351 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 998757 T21396 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 998757 T21351 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 998759 T21442 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@58e5d983[nodes_used_collection_shard2_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 998759 T21351 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica2
   [junit4]   2> 998760 T21351 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica2 address:http://127.0.0.1:51057/_uu/uj collection:nodes_used_collection shard:shard2
   [junit4]   2> 998760 T21411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998761 T21351 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 998764 T21425 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 998766 T21411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998767 T21351 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 998767 T21425 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 998769 T21362 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 998770 T21351 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=0ms
   [junit4]   2> 998770 T21351 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
   [junit4]   2> 998770 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"nodes_used_collection"} current state version: 3
   [junit4]   2> 998770 T21351 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C788 name=nodes_used_collection_shard2_replica2 org.apache.solr.core.SolrCore@89fc779 url=http://127.0.0.1:51057/_uu/uj/nodes_used_collection_shard2_replica2 node=127.0.0.1:51057__uu%2Fuj C788_STATE=coll:nodes_used_collection core:nodes_used_collection_shard2_replica2 props:{core=nodes_used_collection_shard2_replica2, node_name=127.0.0.1:51057__uu%2Fuj, state=down, base_url=http://127.0.0.1:51057/_uu/uj}
   [junit4]   2> 998771 T21351 C788 P51057 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51057/_uu/uj/nodes_used_collection_shard2_replica2/
   [junit4]   2> 998772 T21396 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998772 T21403 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998773 T21362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998773 T21387 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998772 T21351 C788 P51057 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica2 url=http://127.0.0.1:51057/_uu/uj START replicas=[http://127.0.0.1:51088/_uu/uj/nodes_used_collection_shard2_replica1/] nUpdates=100
   [junit4]   2> 998772 T21369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998774 T21351 C788 P51057 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica2 url=http://127.0.0.1:51057/_uu/uj DONE.  We have no versions.  sync failed.
   [junit4]   2> 998776 T21351 C788 P51057 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 998772 T21419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998772 T21411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 998776 T21351 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 998773 T21435 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 998778 T21351 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51057/_uu/uj/nodes_used_collection_shard2_replica2/ shard2
   [junit4]   2> 998778 T21411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 998779 T21351 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard2
   [junit4]   2> 998779 T21411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 998780 T21411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 998780 T21396 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998782 T21411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 998782 T21411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 998783 T21411 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using configuration from collection nodes_used_collection
   [junit4]   2> 998783 T21411 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 998784 T21411 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 998784 T21445 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 998785 T21411 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6f7b130c
   [junit4]   2> 998785 T21425 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 998786 T21363 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51057/_uu/uj",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 998787 T21411 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/data
   [junit4]   2> 998787 T21396 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 998787 T21411 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 998788 T21411 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 998789 T21411 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 998790 T21411 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=1.8212890625, floorSe

[...truncated too long message...]

e collection: awholynewstresscollection_collection0_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=21858, name=Thread-7417, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51063/_uu/uj, http://127.0.0.1:51088/_uu/uj, http://127.0.0.1:51069/_uu/uj, http://127.0.0.1:51075/_uu/uj, http://127.0.0.1:51057/_uu/uj]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C0319ADFA169AF3]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51063/_uu/uj, http://127.0.0.1:51088/_uu/uj, http://127.0.0.1:51069/_uu/uj, http://127.0.0.1:51075/_uu/uj, http://127.0.0.1:51057/_uu/uj]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51088/_uu/uj: Could not fully create collection: awholynewstresscollection_collection2_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=21861, name=Thread-7420, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51063/_uu/uj, http://127.0.0.1:51088/_uu/uj, http://127.0.0.1:51069/_uu/uj, http://127.0.0.1:51075/_uu/uj, http://127.0.0.1:51057/_uu/uj]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C0319ADFA169AF3]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51063/_uu/uj, http://127.0.0.1:51088/_uu/uj, http://127.0.0.1:51069/_uu/uj, http://127.0.0.1:51075/_uu/uj, http://127.0.0.1:51057/_uu/uj]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51088/_uu/uj: Could not fully create collection: awholynewstresscollection_collection5_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 more
   [junit4]   2> 2066034 T21330 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-C0319ADFA169AF3-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9, maxDocsPerChunk=1, blockSize=585), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=9, blockSize=585)), sim=DefaultSimilarity, locale=da_DK, timezone=CAT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=79092040,total=533200896
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, TestSolrJ, TestIndexSearcher, DocValuesTest, HdfsBasicDistributedZk2Test, TestFieldCollectionResource, LeaderFailoverAfterPartitionTest, TestBlendedInfixSuggestions, HdfsDirectoryTest, CurrencyFieldXmlFileTest, TestArbitraryIndexDir, TestRangeQuery, SpatialRPTFieldTypeTest, TestValueSourceCache, BinaryUpdateRequestHandlerTest, URLClassifyProcessorTest, TestManagedResourceStorage, TestSolrDynamicMBean, TestSearchPerf, DistributedFacetPivotSmallAdvancedTest, TestIndexingPerformance, TestCollapseQParserPlugin, DefaultValueUpdateProcessorTest, TestDocBasedVersionConstraints, TestSweetSpotSimilarityFactory, NotRequiredUniqueKeyTest, TestSolrQueryParserResource, TestJoin, SoftAutoCommitTest, TestFieldResource, DistributedIntervalFacetingTest, SSLMigrationTest, TestDistributedGrouping, UnloadDistributedZkTest, HdfsChaosMonkeySafeLeaderTest, TestBinaryField, QueryResultKeyTest, TestCustomSort, SOLR749Test, TestDynamicFieldCollectionResource, TermVectorComponentDistributedTest, TestFastOutputStream, TestAnalyzeInfixSuggestions, SuggesterWFSTTest, HdfsWriteToMultipleCollectionsTest, TestManagedSchemaFieldResource, TestGroupingSearch, PreAnalyzedUpdateProcessorTest, TestPseudoReturnFields, TestPostingsSolrHighlighter, TestWordDelimiterFilterFactory, TestQueryTypes, AnalyticsMergeStrategyTest, CSVRequestHandlerTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSchemaManager, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J3 in 1081.97s, 1 test, 1 error <<< FAILURES!

[...truncated 312 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 460 suites, 1869 tests, 1 error, 57 ignored (10 assumptions)

Total time: 156 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 1357970851 bytes
Compression is 0.0%
Took 3 min 52 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 723 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
Its hitting a new check from mockfilesystem (limits tests to 2048 open
file handles):

https://issues.apache.org/jira/browse/LUCENE-6159

On Sat, Jan 3, 2015 at 1:18 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/723/
>
> 3 tests failed.
> REGRESSION:  org.apache.lucene.search.TestSearcherManager.testSearcherManager
>
> Error Message:
> this IndexWriter is closed
>
> Stack Trace:
> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>         at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
>         at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
>         at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
>         at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
>         at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>         at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
>         at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
>         at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
>         at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
>         at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
>         at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
>         at org.apache.lucene.search.TestSearcherManager.getFinalSearcher(TestSearcherManager.java:68)
>         at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:541)
>         at org.apache.lucene.search.TestSearcherManager.testSearcherManager(TestSearcherManager.java:60)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>         at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>         at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>         at java.nio.file.Files.newOutputStream(Files.java:172)
>         at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>         at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>         at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>         at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>         at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>         at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>         at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>         at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>         at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>         at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>         at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>         at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>         at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>         at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>         at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>         at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>         at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
>
>
> FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager
>
> Error Message:
> 1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager:     1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager:
>    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
>         at sun.misc.Unsafe.park(Native Method)
>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>         at java.lang.Thread.run(Thread.java:745)
>         at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>
>
> FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager
>
> Error Message:
> There are still zombie threads that couldn't be terminated:    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
>         at sun.misc.Unsafe.park(Native Method)
>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>         at java.lang.Thread.run(Thread.java:745)
>         at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>
>
>
>
> Build Log:
> [...truncated 824 lines...]
>    [junit4] Suite: org.apache.lucene.search.TestSearcherManager
>    [junit4]   1> Thread-1587: hit exc
>    [junit4]   1> Thread-1588: hit exc
>    [junit4]   2> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]   2>        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]   2>        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]   2>        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]   2>        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]   2>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]   2>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
>    [junit4]   2> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   2>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   2>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   2>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   2>        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]   2>        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]   2>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   2>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]   2>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]   2>        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]   2>        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]   2>        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]   2>        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]   2>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]   2>        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]   2>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]   2>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
>    [junit4]   1> Thread-1590: hit exc
>    [junit4]   1> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
>    [junit4]   1>        at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
>    [junit4]   1>        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>    [junit4]   1>        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
>    [junit4]   1>        at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
>    [junit4]   1>        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
>    [junit4]   1>        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
>    [junit4]   1>        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
>    [junit4]   1>        at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
>    [junit4]   1>        at org.apache.lucene.search.TestSearcherManager.getCurrentSearcher(TestSearcherManager.java:153)
>    [junit4]   1>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:337)
>    [junit4]   1> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]   1>        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]   1>        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]   1>        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]   1>        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]   1>        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]   1>        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]   1>        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]   1>        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]   1>        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]   1>        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]   1>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]   1>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]   1>        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]   1>        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]   1>        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]   1>        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]   1>        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]   1>        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]   1>        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]   1>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]   1>        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.method=testSearcherManager -Dtests.seed=D6BC19E58A39CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de_CH -Dtests.timezone=Africa/Addis_Ababa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   18.4s J3 | TestSearcherManager.testSearcherManager <<<
>    [junit4]    > Throwable #1: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
>    [junit4]    >        at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
>    [junit4]    >        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
>    [junit4]    >        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
>    [junit4]    >        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
>    [junit4]    >        at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
>    [junit4]    >        at org.apache.lucene.search.TestSearcherManager.getFinalSearcher(TestSearcherManager.java:68)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:541)
>    [junit4]    >        at org.apache.lucene.search.TestSearcherManager.testSearcherManager(TestSearcherManager.java:60)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1817, name=Thread-1589, state=RUNNABLE, group=TGRP-TestSearcherManager]
>    [junit4]    > Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4]    >        at org.apache.lucene.search.TestSearcherManager$2.run(TestSearcherManager.java:134)
>    [junit4]    > Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2697)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:2874)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2841)
>    [junit4]    >        at org.apache.lucene.search.TestSearcherManager$2.run(TestSearcherManager.java:118)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1841, name=Lucene Merge Thread #19, state=RUNNABLE, group=TGRP-TestSearcherManager]
>    [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:609)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:582)
>    [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
>    [junit4]    >        at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:350)
>    [junit4]    >        at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:728)
>    [junit4]    >        at org.apache.lucene.index.IndexFileDeleter.refresh(IndexFileDeleter.java:460)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3551)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:469)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:546)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1815, name=Thread-1587, state=RUNNABLE, group=TGRP-TestSearcherManager]
>    [junit4]    > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:302)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1816, name=Thread-1588, state=RUNNABLE, group=TGRP-TestSearcherManager]
>    [junit4]    > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:302)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1818, name=Thread-1590, state=RUNNABLE, group=TGRP-TestSearcherManager]
>    [junit4]    > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:397)
>    [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
>    [junit4]    >        at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
>    [junit4]    >        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
>    [junit4]    >        at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
>    [junit4]    >        at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
>    [junit4]    >        at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
>    [junit4]    >        at org.apache.lucene.search.TestSearcherManager.getCurrentSearcher(TestSearcherManager.java:153)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:337)
>    [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
>    [junit4]    >        at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
>    [junit4]    >        at java.nio.file.Files.newOutputStream(Files.java:172)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
>    [junit4]    >        at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
>    [junit4]    >        at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
>    [junit4]    >        at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
>    [junit4]    >        at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
>    [junit4]    >        at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
>    [junit4]    >        at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
>    [junit4]    >        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
>    [junit4]    >        at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
>    [junit4]    >        at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
>    [junit4]    >        at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {extra37=PostingsFormat(name=Asserting), extra18=BlockTreeOrds(blocksize=128), extra8=PostingsFormat(name=Asserting), extra27=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra35=FSTOrd50, extra28=FSTOrd50, extra2=FSTOrd50, extra25=BlockTreeOrds(blocksize=128), extra19=PostingsFormat(name=Asserting), docid_int=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra3=BlockTreeOrds(blocksize=128), extra36=BlockTreeOrds(blocksize=128), extra9=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra29=BlockTreeOrds(blocksize=128), title=PostingsFormat(name=Asserting), extra20=FSTOrd50, extra4=PostingsFormat(name=Asserting), extra39=FSTOrd50, date=FSTOrd50, extra14=BlockTreeOrds(blocksize=128), extra5=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra22=PostingsFormat(name=Asserting), extra6=FSTOrd50, extra26=PostingsFormat(name=Asserting), extra10=BlockTreeOrds(blocksize=128), extra0=PostingsFormat(name=Asserting), extra31=FSTOrd50, extra13=FSTOrd50, extra23=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra30=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra16=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra12=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra15=PostingsFormat(name=Asserting), extra7=BlockTreeOrds(blocksize=128), extra1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra11=PostingsFormat(name=Asserting), extra38=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), titleTokenized=BlockTreeOrds(blocksize=128), body=FSTOrd50, extra34=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra33=PostingsFormat(name=Asserting), docid=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra24=FSTOrd50, packID=PostingsFormat(name=Asserting), extra21=BlockTreeOrds(blocksize=128), extra17=FSTOrd50, extra32=BlockTreeOrds(blocksize=128)}, docValues:{docid_intDV=DocValuesFormat(name=Lucene50), titleDV=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=de_CH, timezone=Africa/Addis_Ababa
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=2,free=193111096,total=490209280
>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocumentWriter, TestQueryRescorer, TestSpanSearchEquivalence, TestFlex, TestCachingWrapperFilter, TestStressDeletes, TestBlockPostingsFormat, TestSimpleExplanations, ThrowInUncaught, TestUsageTrackingFilterCachingPolicy, TestDeletionPolicy, TestFrequencyTrackingRingBuffer, TestSumDocFreq, TestMatchAllDocsQuery, TestSnapshotDeletionPolicy, TestLongBitSet, TestMergedIterator, TestTermdocPerf, TestNoDeletionPolicy, TestMultiPhraseQuery, TestIndexReaderClose, TestIndexWriterMaxDocs, TestPositiveScoresOnlyCollector, TestBooleanCoord, TestFilteredQuery, TestBasics, TestConjunctions, TestLookaheadTokenFilter, TestDateSort, TestTwoPhaseCommitTool, TestBooleanQuery, TestBufferedIndexInput, TestMultiThreadTermVectors, TestIntsRef, TestConcurrentMergeScheduler, TestWildcard, TestBooleanQueryVisitSubscorers, TestFieldCacheTermsFilter, TestOperations, TestIndexWriterMerging, TestAutomatonQuery, Nested1, TestSizeBoundedForceMerge, TestReusableStringReader, TestIndexWriterForceMerge, Test2BSortedDocValues, TestSegmentTermEnum, Nested, TestBagOfPositions, TestPhrasePrefixQuery, TestIOUtils, TestIndexWriterOnDiskFull, TestIndexWriter, TestIndexWriterReader, TestCompressingTermVectorsFormat, TestFSTs, TestSearcherManager]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.seed=D6BC19E58A39CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de_CH -Dtests.timezone=Africa/Addis_Ababa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   0.00s J3 | TestSearcherManager (suite) <<<
>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager:
>    [junit4]    >    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
>    [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>    [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
>    [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
>    [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>    [junit4]    >         at java.lang.Thread.run(Thread.java:745)
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>    [junit4]    >    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
>    [junit4]    >         at sun.misc.Unsafe.park(Native Method)
>    [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
>    [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
>    [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
>    [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>    [junit4]    >         at java.lang.Thread.run(Thread.java:745)
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
>    [junit4] Completed on J3 in 41.63s, 9 tests, 3 errors <<< FAILURES!
>
> [...truncated 712 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 410 suites, 3254 tests, 2 suite-level errors, 1 error, 1297 ignored (18 assumptions)
>
> Total time: 27 minutes 34 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 5163429 bytes
> Compression is 0.0%
> Took 1.5 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 723 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/723/

3 tests failed.
REGRESSION:  org.apache.lucene.search.TestSearcherManager.testSearcherManager

Error Message:
this IndexWriter is closed

Stack Trace:
org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
	at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
	at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
	at org.apache.lucene.search.TestSearcherManager.getFinalSearcher(TestSearcherManager.java:68)
	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:541)
	at org.apache.lucene.search.TestSearcherManager.testSearcherManager(TestSearcherManager.java:60)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
	at java.nio.file.Files.newOutputStream(Files.java:172)
	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager:     1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager: 
   1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)




Build Log:
[...truncated 824 lines...]
   [junit4] Suite: org.apache.lucene.search.TestSearcherManager
   [junit4]   1> Thread-1587: hit exc
   [junit4]   1> Thread-1588: hit exc
   [junit4]   2> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]   2> 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]   2> 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]   2> 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]   2> 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]   2> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]   2> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
   [junit4]   2> java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   2> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   2> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   2> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   2> 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]   2> 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]   2> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   2> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]   2> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]   2> 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]   2> 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]   2> 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]   2> 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]   2> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]   2> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
   [junit4]   1> Thread-1590: hit exc
   [junit4]   1> org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
   [junit4]   1> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
   [junit4]   1> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
   [junit4]   1> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
   [junit4]   1> 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
   [junit4]   1> 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
   [junit4]   1> 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
   [junit4]   1> 	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
   [junit4]   1> 	at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
   [junit4]   1> 	at org.apache.lucene.search.TestSearcherManager.getCurrentSearcher(TestSearcherManager.java:153)
   [junit4]   1> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:337)
   [junit4]   1> Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]   1> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]   1> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]   1> 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]   1> 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]   1> 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]   1> 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]   1> 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]   1> 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]   1> 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]   1> 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]   1> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]   1> 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]   1> 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]   1> 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]   1> 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]   1> 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]   1> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]   1> 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]   1> 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]   1> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]   1> 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.method=testSearcherManager -Dtests.seed=D6BC19E58A39CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de_CH -Dtests.timezone=Africa/Addis_Ababa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   18.4s J3 | TestSearcherManager.testSearcherManager <<<
   [junit4]    > Throwable #1: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
   [junit4]    > 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
   [junit4]    > 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
   [junit4]    > 	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
   [junit4]    > 	at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager.getFinalSearcher(TestSearcherManager.java:68)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:541)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager.testSearcherManager(TestSearcherManager.java:60)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1817, name=Thread-1589, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: java.lang.RuntimeException: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager$2.run(TestSearcherManager.java:134)
   [junit4]    > Caused by: java.lang.IllegalStateException: this writer hit an unrecoverable error; cannot commit
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2697)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:2874)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2841)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager$2.run(TestSearcherManager.java:118)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1841, name=Lucene Merge Thread #19, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:609)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:582)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: refusing to delete any files: this IndexWriter hit an unrecoverable exception
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.ensureOpen(IndexFileDeleter.java:350)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.deleteFile(IndexFileDeleter.java:728)
   [junit4]    > 	at org.apache.lucene.index.IndexFileDeleter.refresh(IndexFileDeleter.java:460)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3551)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:469)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:546)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1815, name=Thread-1587, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:302)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1816, name=Thread-1588, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: java.lang.RuntimeException: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:302)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3h.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=1818, name=Thread-1590, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:397)
   [junit4]    > Caused by: org.apache.lucene.store.AlreadyClosedException: this IndexWriter is closed
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:705)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.ensureOpen(IndexWriter.java:719)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.nrtIsCurrent(IndexWriter.java:4449)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:277)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:246)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:123)
   [junit4]    > 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:137)
   [junit4]    > 	at org.apache.lucene.search.SearcherManager.refreshIfNeeded(SearcherManager.java:58)
   [junit4]    > 	at org.apache.lucene.search.ReferenceManager.doMaybeRefresh(ReferenceManager.java:176)
   [junit4]    > 	at org.apache.lucene.search.ReferenceManager.maybeRefresh(ReferenceManager.java:225)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager.getCurrentSearcher(TestSearcherManager.java:153)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$2.run(ThreadedIndexingAndSearchingTestCase.java:337)
   [junit4]    > Caused by: java.nio.file.FileSystemException: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001/TestSearcherManager-001/_3i.tvd: Too many open files
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:49)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newOutputStream(HandleTrackingFS.java:157)
   [junit4]    > 	at java.nio.file.Files.newOutputStream(Files.java:172)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:265)
   [junit4]    > 	at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:214)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73)
   [junit4]    > 	at org.apache.lucene.store.RateLimitedDirectoryWrapper.createOutput(RateLimitedDirectoryWrapper.java:45)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.createOutput(MockDirectoryWrapper.java:607)
   [junit4]    > 	at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.<init>(CompressingTermVectorsWriter.java:226)
   [junit4]    > 	at org.apache.lucene.codecs.compressing.CompressingTermVectorsFormat.vectorsWriter(CompressingTermVectorsFormat.java:98)
   [junit4]    > 	at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat.vectorsWriter(AssertingTermVectorsFormat.java:50)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.initTermVectorsWriter(TermVectorsConsumer.java:88)
   [junit4]    > 	at org.apache.lucene.index.TermVectorsConsumer.finishDocument(TermVectorsConsumer.java:103)
   [junit4]    > 	at org.apache.lucene.index.TermsHash.finishDocument(TermsHash.java:93)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocuments(DocumentsWriterPerThread.java:270)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocuments(DocumentsWriter.java:410)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocuments(IndexWriter.java:1231)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1192)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocuments(IndexWriter.java:1177)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.addDocuments(ThreadedIndexingAndSearchingTestCase.java:99)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase$1.run(ThreadedIndexingAndSearchingTestCase.java:218)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestSearcherManager-D6BC19E58A39CA7-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {extra37=PostingsFormat(name=Asserting), extra18=BlockTreeOrds(blocksize=128), extra8=PostingsFormat(name=Asserting), extra27=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra35=FSTOrd50, extra28=FSTOrd50, extra2=FSTOrd50, extra25=BlockTreeOrds(blocksize=128), extra19=PostingsFormat(name=Asserting), docid_int=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra3=BlockTreeOrds(blocksize=128), extra36=BlockTreeOrds(blocksize=128), extra9=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra29=BlockTreeOrds(blocksize=128), title=PostingsFormat(name=Asserting), extra20=FSTOrd50, extra4=PostingsFormat(name=Asserting), extra39=FSTOrd50, date=FSTOrd50, extra14=BlockTreeOrds(blocksize=128), extra5=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra22=PostingsFormat(name=Asserting), extra6=FSTOrd50, extra26=PostingsFormat(name=Asserting), extra10=BlockTreeOrds(blocksize=128), extra0=PostingsFormat(name=Asserting), extra31=FSTOrd50, extra13=FSTOrd50, extra23=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra30=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra16=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra12=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra15=PostingsFormat(name=Asserting), extra7=BlockTreeOrds(blocksize=128), extra1=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra11=PostingsFormat(name=Asserting), extra38=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), titleTokenized=BlockTreeOrds(blocksize=128), body=FSTOrd50, extra34=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra33=PostingsFormat(name=Asserting), docid=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), extra24=FSTOrd50, packID=PostingsFormat(name=Asserting), extra21=BlockTreeOrds(blocksize=128), extra17=FSTOrd50, extra32=BlockTreeOrds(blocksize=128)}, docValues:{docid_intDV=DocValuesFormat(name=Lucene50), titleDV=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=de_CH, timezone=Africa/Addis_Ababa
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=2,free=193111096,total=490209280
   [junit4]   2> NOTE: All tests run in this JVM: [TestDocumentWriter, TestQueryRescorer, TestSpanSearchEquivalence, TestFlex, TestCachingWrapperFilter, TestStressDeletes, TestBlockPostingsFormat, TestSimpleExplanations, ThrowInUncaught, TestUsageTrackingFilterCachingPolicy, TestDeletionPolicy, TestFrequencyTrackingRingBuffer, TestSumDocFreq, TestMatchAllDocsQuery, TestSnapshotDeletionPolicy, TestLongBitSet, TestMergedIterator, TestTermdocPerf, TestNoDeletionPolicy, TestMultiPhraseQuery, TestIndexReaderClose, TestIndexWriterMaxDocs, TestPositiveScoresOnlyCollector, TestBooleanCoord, TestFilteredQuery, TestBasics, TestConjunctions, TestLookaheadTokenFilter, TestDateSort, TestTwoPhaseCommitTool, TestBooleanQuery, TestBufferedIndexInput, TestMultiThreadTermVectors, TestIntsRef, TestConcurrentMergeScheduler, TestWildcard, TestBooleanQueryVisitSubscorers, TestFieldCacheTermsFilter, TestOperations, TestIndexWriterMerging, TestAutomatonQuery, Nested1, TestSizeBoundedForceMerge, TestReusableStringReader, TestIndexWriterForceMerge, Test2BSortedDocValues, TestSegmentTermEnum, Nested, TestBagOfPositions, TestPhrasePrefixQuery, TestIOUtils, TestIndexWriterOnDiskFull, TestIndexWriter, TestIndexWriterReader, TestCompressingTermVectorsFormat, TestFSTs, TestSearcherManager]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.seed=D6BC19E58A39CA7 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de_CH -Dtests.timezone=Africa/Addis_Ababa -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J3 | TestSearcherManager (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager: 
   [junit4]    >    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1838, name=LuceneTestCase-323-thread-1, state=WAITING, group=TGRP-TestSearcherManager]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6BC19E58A39CA7]:0)
   [junit4] Completed on J3 in 41.63s, 9 tests, 3 errors <<< FAILURES!

[...truncated 712 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 410 suites, 3254 tests, 2 suite-level errors, 1 error, 1297 ignored (18 assumptions)

Total time: 27 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 5163429 bytes
Compression is 0.0%
Took 1.5 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 722 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/722/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at https://127.0.0.1:62231/q_e/o: java.lang.NullPointerException 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:62231/q_e/o: java.lang.NullPointerException

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:449)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:201)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10087 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/init-core-data-001
   [junit4]   2> 312673 T133 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 312674 T133 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /q_e/o
   [junit4]   2> 312681 T133 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 312722 T133 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 312737 T134 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 313139 T133 oasc.ZkTestServer.run start zk server on port:53551
   [junit4]   2> 313152 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 313410 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 313588 T141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ea64c1b name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 313588 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 313589 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 313600 T133 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 313734 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 313735 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 313737 T144 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60bd4767 name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 313738 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 313738 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 313746 T133 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 313752 T133 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 313756 T133 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 313760 T133 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 313766 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 313767 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 313774 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 313775 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 313880 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 313881 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 313886 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 313887 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 313891 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 313892 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 313897 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 313898 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 313903 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 313904 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 313908 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 313909 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 313913 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 313913 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 313918 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 313918 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 313923 T133 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 313923 T133 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 313946 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 313947 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 313950 T147 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76455a0a name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 313950 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 313950 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> Using old style solr.xml
   [junit4]   2> 315745 T133 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 315755 T133 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 315760 T133 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:53559
   [junit4]   2> 315764 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 315765 T133 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 315765 T133 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-002
   [junit4]   2> 315765 T133 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-002/'
   [junit4]   2> 315835 T133 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-002/solr.xml
   [junit4]   2> 315990 T133 oasc.CoreContainer.<init> New CoreContainer 184470565
   [junit4]   2> 315990 T133 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-002/]
   [junit4]   2> 315992 T133 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 315992 T133 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 315992 T133 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 315993 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 315993 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 315994 T133 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 315994 T133 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 315994 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 315995 T133 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 315995 T133 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 315999 T133 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 316003 T133 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 316004 T133 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 316004 T133 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 316005 T133 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53551/solr
   [junit4]   2> 316077 T133 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 316077 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 316078 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316081 T159 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@462cee10 name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316082 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316082 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 316122 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316125 T162 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6181b53f name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316125 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316153 T133 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 316172 T133 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 316184 T133 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 316189 T133 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 316194 T133 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 316226 T133 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 316229 T133 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53559_q_e%2Fo
   [junit4]   2> 316241 T133 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53559_q_e%2Fo
   [junit4]   2> 316247 T133 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 316251 T133 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 316255 T133 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 316280 T133 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:53559_q_e%2Fo
   [junit4]   2> 316281 T133 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 316284 T133 oasc.Overseer.start Overseer (id=93076086524608516-127.0.0.1:53559_q_e%2Fo-n_0000000000) starting
   [junit4]   2> 316310 T133 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 316357 T133 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 316387 T164 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 316388 T133 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 316392 T133 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 316394 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 316437 T163 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 316447 T133 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 316448 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 316530 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 316531 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316533 T169 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e55c179 name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316534 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316534 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 316536 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 316544 T133 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 316545 T133 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 316545 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 316546 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316548 T172 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4767f3e1 name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316548 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316549 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 316568 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 316581 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 0
   [junit4]   2> 316607 T163 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 316678 T163 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 316701 T162 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 316702 T169 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 318137 T133 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 318138 T133 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 318146 T133 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 318151 T133 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:62231
   [junit4]   2> 318154 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 318155 T133 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 318155 T133 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003
   [junit4]   2> 318156 T133 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/'
   [junit4]   2> 318204 T133 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/solr.xml
   [junit4]   2> 318292 T133 oasc.CoreContainer.<init> New CoreContainer 1360022521
   [junit4]   2> 318293 T133 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/]
   [junit4]   2> 318294 T133 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 318294 T133 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 318295 T133 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 318295 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 318295 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 318296 T133 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 318296 T133 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 318297 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 318297 T133 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 318297 T133 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 318301 T133 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 318304 T133 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 318305 T133 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 318305 T133 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 318305 T133 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53551/solr
   [junit4]   2> 318306 T133 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 318306 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 318307 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 318310 T184 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55c3c6be name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 318311 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 318311 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 318314 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 318316 T187 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6692eb4f name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 318317 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 318329 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 319340 T133 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62231_q_e%2Fo
   [junit4]   2> 319342 T133 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62231_q_e%2Fo
   [junit4]   2> 319349 T133 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 319363 T133 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 319364 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 321050 T133 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 321051 T133 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 321060 T133 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 321063 T133 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:27216
   [junit4]   2> 321067 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 321067 T133 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 321068 T133 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004
   [junit4]   2> 321068 T133 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/'
   [junit4]   2> 321117 T133 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/solr.xml
   [junit4]   2> 321208 T133 oasc.CoreContainer.<init> New CoreContainer 1152564872
   [junit4]   2> 321208 T133 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/]
   [junit4]   2> 321294 T133 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 321294 T133 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 321295 T133 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 321295 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 321296 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 321296 T133 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 321296 T133 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 321297 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 321297 T133 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 321297 T133 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 321301 T133 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 321304 T133 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 321305 T133 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 321305 T133 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 321306 T133 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53551/solr
   [junit4]   2> 321306 T133 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 321307 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 321308 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 321311 T200 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28e0db68 name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 321312 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 321313 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 321317 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 321320 T203 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36bd59d4 name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 321320 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 321332 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 322341 T133 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27216_q_e%2Fo
   [junit4]   2> 322343 T133 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27216_q_e%2Fo
   [junit4]   2> 322349 T133 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 322356 T133 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 322357 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 324113 T133 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 324114 T133 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 324122 T133 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 324126 T133 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:43361
   [junit4]   2> 324129 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 324130 T133 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 324130 T133 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005
   [junit4]   2> 324131 T133 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/'
   [junit4]   2> 324178 T133 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/solr.xml
   [junit4]   2> 324263 T133 oasc.CoreContainer.<init> New CoreContainer 1485702028
   [junit4]   2> 324263 T133 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/]
   [junit4]   2> 324265 T133 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 324265 T133 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 324265 T133 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 324266 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 324266 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 324266 T133 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 324267 T133 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 324267 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 324268 T133 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 324268 T133 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 324271 T133 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 324275 T133 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 324275 T133 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 324275 T133 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 324276 T133 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53551/solr
   [junit4]   2> 324276 T133 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 324276 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 324277 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324280 T216 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d8c64bf name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324281 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324281 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 324284 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324286 T219 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@262f801d name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324286 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324297 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 325307 T133 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43361_q_e%2Fo
   [junit4]   2> 325308 T133 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43361_q_e%2Fo
   [junit4]   2> 325316 T133 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 325324 T133 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 325325 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 327053 T133 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 327054 T133 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 327063 T133 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 327066 T133 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:14149
   [junit4]   2> 327070 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 327070 T133 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 327070 T133 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006
   [junit4]   2> 327071 T133 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/'
   [junit4]   2> 327119 T133 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/solr.xml
   [junit4]   2> 327203 T133 oasc.CoreContainer.<init> New CoreContainer 1831728068
   [junit4]   2> 327204 T133 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/]
   [junit4]   2> 327205 T133 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 327205 T133 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 327206 T133 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 327206 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 327206 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 327207 T133 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 327207 T133 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 327208 T133 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 327208 T133 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 327208 T133 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 327212 T133 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=3400000&connTimeout=45000&retry=false
   [junit4]   2> 327215 T133 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 327216 T133 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 327216 T133 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 327216 T133 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53551/solr
   [junit4]   2> 327217 T133 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 327217 T133 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 327218 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327220 T232 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3c720215 name:ZooKeeperConnection Watcher:127.0.0.1:53551 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327221 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327221 T133 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 327224 T133 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327226 T235 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@554e3bde name:ZooKeeperConnection Watcher:127.0.0.1:53551/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327226 T133 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327234 T133 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 328241 T133 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14149_q_e%2Fo
   [junit4]   2> 328243 T133 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14149_q_e%2Fo
   [junit4]   2> 328251 T133 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 328259 T133 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 328260 T133 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 328417 T176 oasha.CollectionsHandler.handleCreateAction Creating Collection : version=2&wt=javabin&name=nodes_used_collection&replicationFactor=2&numShards=2&action=CREATE
   [junit4]   2> 328420 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/collection-queue-work state SyncConnected
   [junit4]   2> 328422 T164 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 328430 T238 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 328439 T238 oasc.OverseerCollectionProcessor.createConfNode Only one config set found in zk - using it:conf1
   [junit4]   2> 328439 T238 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/nodes_used_collection 
   [junit4]   2> 328440 T238 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 328447 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 328450 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"} current state version: 1
   [junit4]   2> 328450 T163 oasco.ClusterStateMutator.createCollection building a new cName: nodes_used_collection
   [junit4]   2> 328453 T187 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328454 T162 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328454 T169 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328454 T219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328454 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328453 T235 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328649 T238 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection nodes_used_collection, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 328650 T238 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:43361_q_e%2Fo
   [junit4]   2> 328674 T238 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:62231_q_e%2Fo
   [junit4]   2> 328676 T238 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:27216_q_e%2Fo
   [junit4]   2> 328678 T238 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:14149_q_e%2Fo
   [junit4]   2> 328824 T197 oasha.CoreAdminHandler.handleCreateAction core create command version=2&wt=javabin&name=nodes_used_collection_shard2_replica1&qt=/admin/cores&numShards=2&action=CREATE&collection.configName=conf1&collection=nodes_used_collection&shard=shard2
   [junit4]   2> 328825 T175 oasha.CoreAdminHandler.handleCreateAction core create command version=2&wt=javabin&name=nodes_used_collection_shard1_replica2&qt=/admin/cores&numShards=2&action=CREATE&collection.configName=conf1&collection=nodes_used_collection&shard=shard1
   [junit4]   2> 328826 T208 oasha.CoreAdminHandler.handleCreateAction core create command version=2&wt=javabin&name=nodes_used_collection_shard1_replica1&qt=/admin/cores&numShards=2&action=CREATE&collection.configName=conf1&collection=nodes_used_collection&shard=shard1
   [junit4]   2> 328836 T225 oasha.CoreAdminHandler.handleCreateAction core create command version=2&wt=javabin&name=nodes_used_collection_shard2_replica2&qt=/admin/cores&numShards=2&action=CREATE&collection.configName=conf1&collection=nodes_used_collection&shard=shard2
   [junit4]   2> 328837 T208 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 328837 T175 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 328836 T197 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 328840 T225 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 328846 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 328846 T175 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 328846 T208 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 328846 T197 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 328847 T225 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 328848 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 4, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:62231_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:62231/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 2
   [junit4]   2> 328864 T163 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:62231_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:62231/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 328874 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:27216_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:27216/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2"} current state version: 2
   [junit4]   2> 328876 T163 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:27216_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:27216/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2"}
   [junit4]   2> 328879 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 328880 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:43361_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43361/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 2
   [junit4]   2> 328881 T163 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:43361_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43361/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 328885 T163 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:14149_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:14149/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2"} current state version: 2
   [junit4]   2> 328886 T163 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:14149_q_e%2Fo",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:14149/q_e/o",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard2"}
   [junit4]   2> 328889 T162 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 328891 T235 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328891 T162 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328891 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328891 T169 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328891 T219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 328891 T187 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 329848 T197 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 329848 T208 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 329848 T225 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 329848 T175 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 329849 T225 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 329849 T208 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 329848 T197 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 329850 T225 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 329849 T175 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 329851 T197 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 329851 T225 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 329851 T208 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 329853 T208 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 329852 T175 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 329853 T175 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 329852 T197 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 329854 T208 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329853 T225 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329855 T208 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 329855 T197 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329855 T175 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 329855 T225 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 329856 T175 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 329856 T197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 329915 T208 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 329917 T225 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 329922 T197 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 329928 T175 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 329957 T225 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 329957 T208 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 329958 T175 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 329958 T197 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 330020 T175 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 330021 T225 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 330021 T197 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 330020 T208 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 330085 T225 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 330086 T208 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 330087 T175 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 330089 T197 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 330152 T208 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 330155 T175 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 330155 T225 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 330157 T197 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 330253 T208 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 330256 T175 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 330257 T225 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 330258 T197 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 330282 T208 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 330286 T175 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 330288 T225 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 330289 T197 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 331087 T175 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 331097 T208 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 331104 T197 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 331108 T225 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 331112 T175 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 331112 T208 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 331115 T175 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 331116 T208 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 331116 T197 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 331120 T197 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 331126 T225 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 331131 T225 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 331143 T208 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331144 T175 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331148 T197 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331150 T208 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331150 T175 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331155 T197 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331156 T208 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331156 T175 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331157 T208 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331159 T208 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331159 T175 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331159 T208 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331160 T225 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331160 T175 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331161 T208 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331161 T175 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331161 T208 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331162 T197 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331162 T208 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using configuration from collection nodes_used_collection
   [junit4]   2> 331163 T175 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331163 T208 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 331164 T175 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331164 T208 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 331164 T175 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using configuration from collection nodes_used_collection
   [junit4]   2> 331166 T197 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331166 T197 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331166 T208 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@51983697
   [junit4]   2> 331166 T197 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331166 T175 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 331167 T225 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331168 T175 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 331168 T197 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331169 T175 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@51983697
   [junit4]   2> 331169 T197 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331170 T197 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using configuration from collection nodes_used_collection
   [junit4]   2> 331170 T197 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 331171 T197 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 331171 T197 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@51983697
   [junit4]   2> 331173 T225 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331174 T225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331175 T225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331175 T225 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331176 T225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331176 T225 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331177 T225 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using configuration from collection nodes_used_collection
   [junit4]   2> 331177 T225 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 331178 T225 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 331178 T225 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@51983697
   [junit4]   2> 331185 T225 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/data
   [junit4]   2> 331186 T225 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 331186 T208 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/data
   [junit4]   2> 331187 T175 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/data
   [junit4]   2> 331186 T197 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/data
   [junit4]   2> 331186 T225 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 331188 T197 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 331188 T225 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-006/nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 331187 T175 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 331187 T208 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 331189 T175 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 331189 T225 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@88c374
   [junit4]   2> 331188 T197 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 331190 T175 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-003/nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 331192 T225 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@f92f42b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@78dc3b09),segFN=segments_1,generation=1}
   [junit4]   2> 331189 T208 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 331192 T225 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331192 T175 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3f265a83
   [junit4]   2> 331192 T197 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-004/nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 331194 T208 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001/tempDir-005/nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 331195 T197 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@254a2d56
   [junit4]   2> 331201 T175 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ff009e0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@681892be),segFN=segments_1,generation=1}
   [junit4]   2> 331202 T175 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331201 T208 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5b7fb110
   [junit4]   2> 331202 T197 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1c41c05e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@20bc69d0),segFN=segments_1,generation=1}
   [junit4]   2> 331203 T208 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7987141d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1339ba40),segFN=segments_1,generation=1}
   [junit4]   2> 331203 T197 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331204 T208 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331222 T225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 331224 T225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 331225 T225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 331226 T225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331227 T225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331227 T225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 331229 T225 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331229 T197 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 331230 T197 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 331230 T225 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331230 T197 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 331231 T225 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 331232 T175 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 331232 T197 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331233 T175 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 331234 T208 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 331234 T197 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331235 T208 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 331235 T197 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 331236 T208 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 331234 T175 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 331236 T208 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331236 T197 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331237 T208 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331237 T175 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331238 T208 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 331238 T197 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331239 T175 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331240 T208 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331240 T197 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 331241 T208 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331240 T175 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 331241 T208 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 331242 T175 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331243 T175 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331243 T175 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 331248 T175 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331248 T225 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331248 T208 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331250 T175 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331248 T197 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331251 T225 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331251 T175 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331251 T208 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331252 T197 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331252 T225 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331252 T175 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331253 T208 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 331253 T197 oasc.RequestHandlers.initHandlersFromConfig created

[...truncated too long message...]

j.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:14149/q_e/o: Could not fully create collection: awholynewstresscollection_collection1_5
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=821, name=Thread-309, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:62231/q_e/o, https://127.0.0.1:53559/q_e/o, https://127.0.0.1:14149/q_e/o, https://127.0.0.1:27216/q_e/o, https://127.0.0.1:43361/q_e/o]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E85A8117DC3C9366]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:62231/q_e/o, https://127.0.0.1:53559/q_e/o, https://127.0.0.1:14149/q_e/o, https://127.0.0.1:27216/q_e/o, https://127.0.0.1:43361/q_e/o]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:14149/q_e/o: Could not fully create collection: awholynewstresscollection_collection4_7
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=822, name=Thread-310, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:62231/q_e/o, https://127.0.0.1:53559/q_e/o, https://127.0.0.1:14149/q_e/o, https://127.0.0.1:27216/q_e/o, https://127.0.0.1:43361/q_e/o]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E85A8117DC3C9366]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:62231/q_e/o, https://127.0.0.1:53559/q_e/o, https://127.0.0.1:14149/q_e/o, https://127.0.0.1:27216/q_e/o, https://127.0.0.1:43361/q_e/o]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:14149/q_e/o: Could not fully create collection: awholynewstresscollection_collection5_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 more
   [junit4]   2> 1463213 T133 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-E85A8117DC3C9366-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), a_i1=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), other_tl1=Lucene50(blocksize=128), intDefault=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), text=BlockTreeOrds(blocksize=128), range_facet_l=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=ca, timezone=Mideast/Riyadh89
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=347501960,total=513277952
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSynonymFilterFactory, HdfsCollectionsAPIDistributedZkTest, IndexSchemaRuntimeFieldTest, SolrIndexConfigTest, DocValuesMultiTest, TestStressRecovery, TestIntervalFaceting, URLClassifyProcessorTest, TestNoOpRegenerator, HdfsBasicDistributedZkTest, TestCursorMarkWithoutUniqueKey, AddBlockUpdateTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J3 in 1150.62s, 1 test, 1 error <<< FAILURES!

[...truncated 1118 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 458 suites, 1858 tests, 1 error, 57 ignored (10 assumptions)

Total time: 222 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 913464105 bytes
Compression is 0.0%
Took 2 min 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 721 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/721/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
IOException occured when talking to server at: http://127.0.0.1:55933/db_cgh/b/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:55933/db_cgh/b/collection1
	at __randomizedtesting.SeedInfo.seed([F3AAE198B3ED19D5:724C6F80C4B279E9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:302)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:413)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:143)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([F3AAE198B3ED19D5:A27E1818029E2972]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9628 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001
   [junit4]   2> 64145 T155 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 64147 T155 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 64148 T155 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 64149 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 64150 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 64151 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 64256 T155 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 64380 T155 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 64476 T155 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 64477 T155 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 64511 T155 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 64875 T155 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 64890 T155 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 64893 T155 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 64914 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 64920 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 64924 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 64925 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 64925 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 64925 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 64926 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 64926 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 64927 T155 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 64928 T155 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 64928 T155 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 65053 T155 oasc.CoreContainer.<init> New CoreContainer 643771460
   [junit4]   2> 65054 T155 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 65055 T155 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 65055 T155 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 65056 T155 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 65056 T155 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 65056 T155 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 65057 T155 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 65057 T155 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 65057 T155 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 65057 T155 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 65058 T155 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 65058 T155 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=false
   [junit4]   2> 65059 T155 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 65059 T155 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 65059 T155 oasc.CoreContainer.load Host Name: 
   [junit4]   2> 65065 T156 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 65067 T156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 65067 T156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 65068 T156 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 65128 T156 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 65164 T156 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 65232 T156 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 65233 T156 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 65260 T156 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 65614 T156 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 65622 T156 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 65625 T156 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 65645 T156 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 65649 T156 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 65653 T156 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 65654 T156 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 65654 T156 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 65654 T156 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 65655 T156 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 65655 T156 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 65656 T156 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 65656 T156 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 65656 T156 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/
   [junit4]   2> 65657 T156 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a463e88
   [junit4]   2> 65678 T156 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001
   [junit4]   2> 65678 T156 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index/
   [junit4]   2> 65678 T156 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 65679 T156 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index
   [junit4]   2> 65679 T156 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=11.154296875, floorSegmentMB=1.5830078125, forceMergeDeletesPctAllowed=21.419693259561356, segmentsPerTier=17.0, maxCFSSegmentSizeMB=0.4169921875, noCFSRatio=0.21761537193893893
   [junit4]   2> 65680 T156 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_1,generation=1}
   [junit4]   2> 65681 T156 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 65757 T156 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 66023 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 66023 T156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 66024 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 66024 T156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 66025 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 66025 T156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 66026 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 66053 T156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 66054 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 66055 T156 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 66055 T156 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 66056 T156 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 66057 T156 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66058 T156 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66058 T156 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66059 T156 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 66060 T156 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 66061 T156 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 66061 T156 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 66062 T156 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 66063 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 66064 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 66065 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 66066 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 66067 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 66068 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 66068 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 66069 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 66070 T156 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 66243 T156 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 66245 T156 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 66246 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 66247 T156 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 66247 T156 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 66248 T156 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 66249 T156 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 66249 T156 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66250 T156 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66251 T156 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66252 T156 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66252 T156 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66253 T156 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66254 T156 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66255 T156 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 66303 T156 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 66353 T156 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 66353 T156 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 66354 T156 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 66379 T156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66381 T156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66384 T156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66386 T156 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 66395 T156 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 66397 T156 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 66398 T156 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 66399 T156 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 66400 T156 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7533345309929097]
   [junit4]   2> 66400 T156 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_1,generation=1}
   [junit4]   2> 66401 T156 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 66401 T156 oass.SolrIndexSearcher.<init> Opening Searcher@17ef9400[collection1] main
   [junit4]   2> 66421 T156 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 66422 T156 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 66422 T156 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 66422 T156 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 66698 T156 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 67054 T156 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 67055 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 67057 T156 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 67058 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 67058 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 67058 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 67061 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 67063 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 67064 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 67066 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 67067 T157 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 67069 T157 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17ef9400[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 67079 T155 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 67087 T155 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 67087 T155 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 67089 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 67089 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 67090 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 67188 T155 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 67229 T155 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 67287 T155 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 67288 T155 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 67310 T155 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 67676 T155 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 67696 T155 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 67700 T155 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 67733 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67740 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67745 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 67747 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 67747 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 67748 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 67749 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 67749 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 67750 T155 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 67751 T155 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/
   [junit4]   2> 67752 T155 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a463e88
   [junit4]   2> 67753 T155 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index/
   [junit4]   2> 67753 T155 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 67763 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 67764 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 67764 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 67765 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 67766 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 67766 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 67767 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 67768 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 67768 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 67769 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 67770 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 67770 T155 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 67772 T155 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 67773 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 67774 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 67775 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 67776 T155 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 67777 T155 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 67778 T155 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 67779 T155 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 67780 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 67782 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 67783 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 67784 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 67785 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 67786 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 67787 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 67788 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 67789 T155 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 67790 T155 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 67791 T155 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 67792 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 67793 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 67794 T155 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 67795 T155 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 67796 T155 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 67798 T155 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67799 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67800 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67802 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67803 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67804 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67805 T155 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67806 T155 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 67807 T155 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 67808 T155 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 67809 T155 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 67810 T155 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 67838 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67840 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67843 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67845 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67848 T155 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 67850 T155 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 67858 T155 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 67859 T155 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 67861 T155 oass.SolrIndexSearcher.<init> Opening Searcher@6820377b[collection1] main
   [junit4]   2> 67861 T155 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 67862 T155 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 67862 T155 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 67863 T155 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 67867 T155 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 67889 T155 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 67890 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 67892 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 67893 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 67893 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 67894 T155 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 67894 T155 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 67895 T155 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 67895 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 67896 T155 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=55.431640625, floorSegmentMB=0.9873046875, forceMergeDeletesPctAllowed=16.088752577698678, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 67897 T155 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_1,generation=1}
   [junit4]   2> 67897 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 67898 T155 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 67899 T155 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 67899 T155 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 67900 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 67900 T155 oass.SolrIndexSearcher.<init> Opening Searcher@6dfadcb[collection1] main
   [junit4]   2> 67901 T155 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 67901 T155 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1b9865f9
   [junit4]   2> 67902 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 67902 T155 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 67903 T155 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 67904 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 67904 T159 oasc.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo WARN Could not getStatistics on info bean org.apache.solr.search.SolrIndexSearcher org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed
   [junit4]   2> 	at org.apache.lucene.index.IndexReader.ensureOpen(IndexReader.java:274)
   [junit4]   2> 	at org.apache.lucene.index.StandardDirectoryReader.getVersion(StandardDirectoryReader.java:326)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.solr.search.SolrIndexSearcher.getStatistics(SolrIndexSearcher.java:2283)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo(JmxMonitoredMap.java:257)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap.put(JmxMonitoredMap.java:150)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap.put(JmxMonitoredMap.java:51)
   [junit4]   2> 	at org.apache.solr.search.SolrIndexSearcher.register(SolrIndexSearcher.java:420)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.registerSearcher(SolrCore.java:1977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.access$100(SolrCore.java:155)
   [junit4]   2> 	at org.apache.solr.core.SolrCore$7.call(SolrCore.java:1844)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 67908 T159 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6820377b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 67908 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 67908 T155 oasc.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo WARN Could not getStatistics on info bean org.apache.solr.search.SolrIndexSearcher org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed
   [junit4]   2> 	at org.apache.lucene.index.IndexReader.ensureOpen(IndexReader.java:274)
   [junit4]   2> 	at org.apache.lucene.index.StandardDirectoryReader.getVersion(StandardDirectoryReader.java:326)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.solr.search.SolrIndexSearcher.getStatistics(SolrIndexSearcher.java:2283)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo(JmxMonitoredMap.java:257)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.objectInstancesFromFilteredNamedObjects(DefaultMBeanServerInterceptor.java:1556)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryMBeansImpl(DefaultMBeanServerInterceptor.java:513)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.queryNames(DefaultMBeanServerInterceptor.java:526)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.queryNames(JmxMBeanServer.java:619)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap.clear(JmxMonitoredMap.java:120)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1161)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.registerCore(CoreContainer.java:471)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:606)
   [junit4]   2> 	at org.apache.solr.update.SoftAutoCommitTest.setUp(SoftAutoCommitTest.java:87)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:861)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 67910 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 67911 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 67911 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 67913 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 67915 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 67917 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 67919 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 67921 T159 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 67922 T159 oasc.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo WARN Could not getStatistics on info bean org.apache.solr.search.SolrIndexSearcher org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed
   [junit4]   2> 	at org.apache.lucene.index.IndexReader.ensureOpen(IndexReader.java:274)
   [junit4]   2> 	at org.apache.lucene.index.StandardDirectoryReader.getVersion(StandardDirectoryReader.java:326)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.lucene.index.FilterDirectoryReader.getVersion(FilterDirectoryReader.java:119)
   [junit4]   2> 	at org.apache.solr.search.SolrIndexSearcher.getStatistics(SolrIndexSearcher.java:2283)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap$SolrDynamicMBean.getMBeanInfo(JmxMonitoredMap.java:257)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap.put(JmxMonitoredMap.java:150)
   [junit4]   2> 	at org.apache.solr.core.JmxMonitoredMap.put(JmxMonitoredMap.java:51)
   [junit4]   2> 	at org.apache.solr.search.SolrIndexSearcher.register(SolrIndexSearcher.java:420)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.registerSearcher(SolrCore.java:1977)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.access$100(SolrCore.java:155)
   [junit4]   2> 	at org.apache.solr.core.SolrCore$7.call(SolrCore.java:1844)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 67925 T159 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6dfadcb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C16 name=collection1 org.apache.solr.core.SolrCore@61c29752
   [junit4]   2> 68365 T155 C16 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1489130970508427264)]} 0 440
   [junit4]   2> 68866 T161 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 69396 T155 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=F3AAE198B3ED19D5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_HK -Dtests.timezone=Asia/Jerusalem -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 2.41s J3 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F3AAE198B3ED19D5:A27E1818029E2972]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 69496 T155 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 69497 T155 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 69499 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 69500 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 69501 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 69566 T160 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 69698 T155 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 69748 T155 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 69843 T155 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 69845 T155 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 69931 T161 oass.SolrIndexSearcher.<init> Opening Searcher@a2bc760[collection1] main
   [junit4]   2> 69935 T155 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 69950 T161 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 69951 T160 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_2,generation=2}
   [junit4]   2> 69952 T160 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 69953 T159 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a2bc760[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c1)))}
   [junit4]   2> 69954 T160 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 69955 T160 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 70450 T155 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 70472 T155 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 70476 T155 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 70509 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70516 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70532 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70534 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70534 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70535 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70536 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70536 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70537 T155 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 70538 T155 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/
   [junit4]   2> 70539 T155 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a463e88
   [junit4]   2> 70540 T155 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index/
   [junit4]   2> 70540 T155 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 70550 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 70551 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 70552 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 70552 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 70553 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70553 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70554 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 70555 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 70555 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 70556 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70557 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70558 T155 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 70559 T155 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70560 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70561 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70562 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 70563 T155 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 70564 T155 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 70568 T155 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 70569 T155 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 70570 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 70572 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 70573 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 70574 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 70575 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 70576 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 70577 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 70578 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 70579 T155 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 70580 T155 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 70582 T155 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 70583 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 70584 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 70585 T155 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 70585 T155 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 70586 T155 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 70587 T155 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70588 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70589 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70590 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70591 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70593 T155 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70594 T155 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70595 T155 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 70596 T155 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 70597 T155 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 70598 T155 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 70599 T155 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 70629 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70632 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70634 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70637 T155 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70640 T155 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 70642 T155 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 70643 T155 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 70644 T155 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 70646 T155 oass.SolrIndexSearcher.<init> Opening Searcher@2f56c2b0[collection1] main
   [junit4]   2> 70647 T155 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 70648 T155 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 70649 T155 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 70649 T155 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 70653 T155 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 70672 T155 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 70673 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 70675 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 70676 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 70676 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 70677 T155 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 70678 T155 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 70677 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 70678 T155 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 70680 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 70680 T155 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8296928213579646]
   [junit4]   2> 70681 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 70683 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 70684 T162 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 70684 T155 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(BaseDirectoryWrapper(RAMDirectory@56cdfc29 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d661444)),segFN=segments_2,generation=2}
   [junit4]   2> 70685 T155 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 70686 T155 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 70687 T162 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2f56c2b0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c1)))}
   [junit4]   2> 70693 T155 oass.SolrIndexSearcher.<init> Opening Searcher@3d63ba6[collection1] main
   [junit4]   2> 70694 T155 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 70695 T155 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@61c29752
   [junit4]   2> 70695 T155 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 70696 T162 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3d63ba6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c1)))}
   [junit4]   2> 70696 T155 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C17 name=collection1 org.apache.solr.core.SolrCore@2959466d
   [junit4]   2> 70722 T155 C17 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1489130973382574080)]} 0 5
   [junit4]   2> 70727 T155 C17 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1489130973390962688)]} 0 2
   [junit4]   2> 70731 T155 C17 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1489130973396205568)]} 0 1
   [junit4]   2> 70735 T155 C17 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1489130973400399872)]} 0 1
   [junit4]   2> 70739 T155 C17 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1489130973404594176)]} 0 1
   [junit4]   2> 70742 T155 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 70755 T155 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 70756 T155 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 70757 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 70758 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 70759 T155 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 70909 T155 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 70952 T155 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 71009 T155 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 71010 T155 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 71025 T155 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 71223 T164 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 71355 T155 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 71368 T155 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 71371 T164 oass.SolrIndexSearcher.<init> Opening Searcher@78620b1c[collection1] main
   [junit4]   2> 71371 T155 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 71372 T164 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 71374 T162 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@78620b1c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.0.0):c1) Uninverting(_1(5.0.0):c5)))}
   [junit4]   2> 71392 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 71415 T155 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 71424 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 71425 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 71426 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 71426 T155 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 71427 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 71427 T155 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 71427 T155 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 71428 T155 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/
   [junit4]   2> 71428 T155 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a463e88
   [junit4]   2> 71429 T155 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest-F3AAE198B3ED19D5-001/init-core-data-001/index/
   [junit4]   2> 71429 T155 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 71436 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 71436 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 71436 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 71437 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 71437 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 71437 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 71438 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 71438 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 71439 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 71439 T155 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 71440 T155 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 71440 T155 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 71441 T155 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 71442 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 71442 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 71443 T155 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 71444 T155 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 71444 T155 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 71446 T155 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 71446 T155 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 71447 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 71448 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 71449 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 71449 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 71450 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 71451 T155 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 71451 T155 

[...truncated too long message...]

sts-5.x/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.FullSolrCloudDistribCmdsTest-F3AAE198B3ED19D5-001/tempDir-007/collection2_shard4_replica3/data/index
   [junit4]   2> 1592574 T6170 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/db_cgh/b,null}
   [junit4]   2> 1592775 T6170 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 45707
   [junit4]   2> 1592776 T6170 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1592777 T6170 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:55930 55930
   [junit4]   2> 1592885 T6171 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:55930 55930
   [junit4]   2> 1592888 T6171 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		14	/solr/aliases.json
   [junit4]   2> 		14	/solr/clusterstate.json
   [junit4]   2> 		11	/solr/configs/conf1
   [junit4]   2> 		6	/solr/collections/collection1/state.json
   [junit4]   2> 		5	/solr/collections/collection2/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/collection2/leader_elect/shard2/election/93070773308620813-core_node9-n_0000000000
   [junit4]   2> 		2	/solr/overseer_elect/election/93070773308620809-127.0.0.1:55941_db_cgh%2Fb-n_0000000002
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		14	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=FullSolrCloudDistribCmdsTest -Dtests.method=testDistribSearch -Dtests.seed=F3AAE198B3ED19D5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ca -Dtests.timezone=Portugal -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    187s J3 | FullSolrCloudDistribCmdsTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:55933/db_cgh/b/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F3AAE198B3ED19D5:724C6F80C4B279E9]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:302)
   [junit4]    > 	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:223)
   [junit4]    > 	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:165)
   [junit4]    > 	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:413)
   [junit4]    > 	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:143)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketException: Broken pipe
   [junit4]    > 	at java.net.SocketOutputStream.socketWrite0(Native Method)
   [junit4]    > 	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
   [junit4]    > 	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
   [junit4]    > 	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
   [junit4]    > 	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 49 more
   [junit4]   2> 1592904 T6170 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.FullSolrCloudDistribCmdsTest-F3AAE198B3ED19D5-001
   [junit4]   2> 187485 T6169 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ca, timezone=Portugal
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=165276688,total=467140608
   [junit4]   2> NOTE: All tests run in this JVM: [AssignTest, RemoteQueryErrorTest, SoftAutoCommitTest, TestDistributedGrouping, HdfsRecoveryZkTest, DistributedFacetPivotWhiteBoxTest, BinaryUpdateRequestHandlerTest, TestXIncludeConfig, QueryResultKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestConfigOverlay, TestUniqueKeyFieldResource, TestJoin, TestRawResponseWriter, TestCloudSchemaless, SolrCoreTest, TestRangeQuery, TestFastOutputStream, TestClusterStateMutator, TestRestManager, HdfsSyncSliceTest, LeaderInitiatedRecoveryOnCommitTest, DocValuesTest, SolrIndexConfigTest, ZkNodePropsTest, UpdateRequestProcessorFactoryTest, TestCursorMarkWithoutUniqueKey, SolrXmlInZkTest, HdfsBasicDistributedZk2Test, TestManagedSynonymFilterFactory, TestFastWriter, TestInitParams, TestManagedResource, TestLMJelinekMercerSimilarityFactory, TestDistributedMissingSort, TestConfigReload, TestQueryTypes, TestDynamicLoading, XsltUpdateRequestHandlerTest, TestBinaryField, DistributedDebugComponentTest, TestPseudoReturnFields, RAMDirectoryFactoryTest, TestHighlightDedupGrouping, OverseerStatusTest, IndexSchemaRuntimeFieldTest, TestIndexingPerformance, TestFiltering, TestSystemIdResolver, TestSolrIndexConfig, JsonLoaderTest, TestPivotHelperCode, TestGroupingSearch, CoreAdminHandlerTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, ShardRoutingTest, FullSolrCloudDistribCmdsTest]
   [junit4] Completed on J3 in 188.28s, 1 test, 1 error <<< FAILURES!

[...truncated 867 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 458 suites, 1858 tests, 1 error, 1 failure, 57 ignored (10 assumptions)

Total time: 163 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 786481184 bytes
Compression is 0.0%
Took 2 min 16 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 720 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/720/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.AliasIntegrationTest.testDistribSearch

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([6A3703BC82601E40:EBD18DA4F53F7E7C]:0)
	at org.apache.solr.cloud.AliasIntegrationTest.doTest(AliasIntegrationTest.java:232)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:24500: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:24500: Error CREATEing SolrCore 'halfcollection_shard1_replica1': Unable to create core [halfcollection_shard1_replica1] Caused by: Could not get shard id for core: halfcollection_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:449)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:201)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:868)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9892 lines...]
   [junit4] Suite: org.apache.solr.cloud.AliasIntegrationTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/init-core-data-001
   [junit4]   2> 300267 T1319 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 300267 T1319 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_l/h
   [junit4]   2> 300274 T1319 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 300276 T1319 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 300276 T1320 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 300376 T1319 oasc.ZkTestServer.run start zk server on port:13193
   [junit4]   2> 300377 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300378 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300383 T1327 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b70ff2b name:ZooKeeperConnection Watcher:127.0.0.1:13193 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300383 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300384 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300384 T1319 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 300388 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300389 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300391 T1330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@153f3b33 name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300391 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300391 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300392 T1319 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 300395 T1319 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 300396 T1319 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 300398 T1319 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 300400 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 300400 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 300403 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 300404 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 300507 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 300508 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 300510 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 300511 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 300513 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 300513 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 300515 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 300516 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 300518 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 300519 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 300521 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 300521 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 300523 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 300524 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 300526 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 300526 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 300529 T1319 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 300529 T1319 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 301696 T1319 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 301702 T1319 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13196
   [junit4]   2> 301703 T1319 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 301703 T1319 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 301703 T1319 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002
   [junit4]   2> 301704 T1319 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/'
   [junit4]   2> 301734 T1319 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/solr.xml
   [junit4]   2> 301802 T1319 oasc.CoreContainer.<init> New CoreContainer 1775337738
   [junit4]   2> 301802 T1319 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/]
   [junit4]   2> 301804 T1319 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 301804 T1319 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 301804 T1319 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 301805 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 301805 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 301805 T1319 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 301806 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 301806 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 301807 T1319 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 301807 T1319 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 301808 T1319 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 301808 T1319 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 301809 T1319 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 301809 T1319 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 301809 T1319 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13193/solr
   [junit4]   2> 301810 T1319 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 301810 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 301811 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 301813 T1342 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62736f7e name:ZooKeeperConnection Watcher:127.0.0.1:13193 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 301814 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 301815 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 301817 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 301819 T1345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19c36aad name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 301819 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 301821 T1319 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 301825 T1319 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 301827 T1319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 301829 T1319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 301831 T1319 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 301833 T1319 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 301835 T1319 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13196__l%2Fh
   [junit4]   2> 301835 T1319 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13196__l%2Fh
   [junit4]   2> 301837 T1319 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 301839 T1319 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 301854 T1319 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 301858 T1319 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:13196__l%2Fh
   [junit4]   2> 301859 T1319 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 301861 T1319 oasc.Overseer.start Overseer (id=93064405938733059-127.0.0.1:13196__l%2Fh-n_0000000000) starting
   [junit4]   2> 301865 T1319 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 301871 T1319 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 301873 T1347 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 301873 T1319 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 301875 T1319 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 301877 T1319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 301880 T1346 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 301884 T1349 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 301885 T1349 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 301886 T1349 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 301886 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 301888 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13196/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13196__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 301889 T1346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13196/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13196__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 301889 T1346 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 301889 T1346 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 301891 T1345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 302886 T1349 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 302887 T1349 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 302888 T1349 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 302888 T1349 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 302889 T1349 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 302889 T1349 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/collection1/'
   [junit4]   2> 302890 T1349 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 302891 T1349 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 302892 T1349 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 302926 T1349 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 302928 T1349 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 302962 T1349 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 303009 T1349 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 303110 T1349 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 303119 T1349 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 303507 T1349 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 303520 T1349 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 303523 T1349 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 303535 T1349 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303540 T1349 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303544 T1349 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303544 T1349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303546 T1349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303546 T1349 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303547 T1349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303547 T1349 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303547 T1349 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 303548 T1349 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 303548 T1349 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-002/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/control/data/
   [junit4]   2> 303548 T1349 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16552add
   [junit4]   2> 303549 T1349 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/control/data
   [junit4]   2> 303549 T1349 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/control/data/index/
   [junit4]   2> 303550 T1349 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 303550 T1349 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/control/data/index
   [junit4]   2> 303550 T1349 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=65.9404296875, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=11.134789867608193, segmentsPerTier=48.0, maxCFSSegmentSizeMB=0.9462890625, noCFSRatio=1.0
   [junit4]   2> 303551 T1349 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@32e149f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64becf92),segFN=segments_1,generation=1}
   [junit4]   2> 303552 T1349 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303556 T1349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 303556 T1349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 303557 T1349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 303557 T1349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303557 T1349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303557 T1349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 303558 T1349 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303558 T1349 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303558 T1349 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 303559 T1349 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 303560 T1349 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 303560 T1349 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 303560 T1349 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 303561 T1349 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 303561 T1349 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 303562 T1349 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 303562 T1349 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 303563 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 303564 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 303565 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 303565 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 303566 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 303566 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 303567 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 303567 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 303568 T1349 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 303568 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 303569 T1349 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 303587 T1349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303589 T1349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303591 T1349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303593 T1349 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303596 T1349 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 303599 T1349 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 303599 T1349 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 303600 T1349 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=8, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.83480000822237]
   [junit4]   2> 303601 T1349 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@32e149f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64becf92),segFN=segments_1,generation=1}
   [junit4]   2> 303601 T1349 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303602 T1349 oass.SolrIndexSearcher.<init> Opening Searcher@1dd288fd[collection1] main
   [junit4]   2> 303602 T1349 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 303603 T1349 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 303603 T1349 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 303604 T1349 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 303604 T1349 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 303604 T1349 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303605 T1349 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 303605 T1349 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303605 T1349 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 303606 T1349 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 303606 T1349 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 303606 T1350 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1dd288fd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 303607 T1353 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13196/_l/h collection:control_collection shard:shard1
   [junit4]   2> 303607 T1319 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 303608 T1319 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 303608 T1353 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 303608 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 303609 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 303611 T1356 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44ade290 name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 303611 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 303611 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 303613 T1319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 303614 T1353 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 303615 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 303616 T1353 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 303616 T1353 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 303616 T1319 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 303616 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C464 name=collection1 org.apache.solr.core.SolrCore@18b3ef79 url=http://127.0.0.1:13196/_l/h/collection1 node=127.0.0.1:13196__l%2Fh C464_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:13196/_l/h, node_name=127.0.0.1:13196__l%2Fh, core=collection1, state=down}
   [junit4]   2> 303616 T1353 C464 P13196 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13196/_l/h/collection1/
   [junit4]   2> 303616 T1319 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 303617 T1353 C464 P13196 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 303617 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 303618 T1345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 303618 T1356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 303618 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 303618 T1353 C464 P13196 oasc.SyncStrategy.syncToMe http://127.0.0.1:13196/_l/h/collection1/ has no replicas
   [junit4]   2> 303620 T1353 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13196/_l/h/collection1/ shard1
   [junit4]   2> 303620 T1353 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 303621 T1359 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3644fdd4 name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 303621 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 303622 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 303624 T1360 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 303626 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":2} current state version: 2
   [junit4]   2> 303626 T1346 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 303631 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13196/_l/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 303632 T1346 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 303634 T1356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 303636 T1360 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 303637 T1356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 303676 T1353 oasc.ZkController.register We are http://127.0.0.1:13196/_l/h/collection1/ and leader is http://127.0.0.1:13196/_l/h/collection1/
   [junit4]   2> 303677 T1353 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13196/_l/h
   [junit4]   2> 303677 T1353 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 303677 T1353 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 303678 T1353 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 303679 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 303681 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13196/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13196__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 303681 T1346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13196/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13196__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 303785 T1345 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 303785 T1356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 304764 T1319 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 304765 T1319 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 304769 T1319 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13220
   [junit4]   2> 304769 T1319 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 304769 T1319 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 304770 T1319 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003
   [junit4]   2> 304770 T1319 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/'
   [junit4]   2> 304800 T1319 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/solr.xml
   [junit4]   2> 304853 T1319 oasc.CoreContainer.<init> New CoreContainer 1526381141
   [junit4]   2> 304854 T1319 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/]
   [junit4]   2> 304855 T1319 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 304855 T1319 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 304855 T1319 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 304855 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 304856 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 304856 T1319 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 304856 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 304856 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 304857 T1319 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 304857 T1319 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 304857 T1319 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 304858 T1319 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 304858 T1319 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 304858 T1319 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 304858 T1319 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13193/solr
   [junit4]   2> 304859 T1319 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 304859 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 304859 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 304861 T1372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4eb3255a name:ZooKeeperConnection Watcher:127.0.0.1:13193 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 304862 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 304863 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 304865 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 304866 T1375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b946dac name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 304866 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 304871 T1319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 305875 T1319 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13220__l%2Fh
   [junit4]   2> 305876 T1319 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13220__l%2Fh
   [junit4]   2> 305880 T1319 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 305887 T1376 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 305888 T1376 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 305889 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 305890 T1376 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 305890 T1376 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 305890 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13220/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13220__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 305890 T1376 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 305891 T1346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13220/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13220__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 305891 T1346 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 305891 T1346 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 305891 T1376 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 305892 T1376 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 305893 T1346 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 305893 T1375 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 305894 T1375 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 306892 T1376 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 306893 T1376 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 306894 T1376 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 306894 T1376 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 306894 T1376 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 306895 T1376 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/collection1/'
   [junit4]   2> 306896 T1376 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 306897 T1376 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 306898 T1376 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 306934 T1376 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 306937 T1376 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 306971 T1376 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 307018 T1376 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 307119 T1376 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 307129 T1376 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 307506 T1376 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 307519 T1376 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 307522 T1376 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 307534 T1376 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 307539 T1376 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 307543 T1376 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 307544 T1376 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 307545 T1376 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 307546 T1376 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 307547 T1376 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 307547 T1376 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 307547 T1376 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 307547 T1376 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 307548 T1376 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-003/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty1/
   [junit4]   2> 307548 T1376 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16552add
   [junit4]   2> 307549 T1376 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty1
   [junit4]   2> 307549 T1376 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty1/index/
   [junit4]   2> 307549 T1376 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 307550 T1376 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty1/index
   [junit4]   2> 307550 T1376 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=65.9404296875, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=11.134789867608193, segmentsPerTier=48.0, maxCFSSegmentSizeMB=0.9462890625, noCFSRatio=1.0
   [junit4]   2> 307551 T1376 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ce29336 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f2efe47),segFN=segments_1,generation=1}
   [junit4]   2> 307551 T1376 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 307557 T1376 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 307557 T1376 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 307557 T1376 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 307558 T1376 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 307558 T1376 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 307558 T1376 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 307559 T1376 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 307559 T1376 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 307559 T1376 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 307560 T1376 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 307561 T1376 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 307561 T1376 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 307562 T1376 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 307562 T1376 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 307563 T1376 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 307563 T1376 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 307564 T1376 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 307565 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 307566 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 307566 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 307567 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 307567 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 307568 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 307569 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 307569 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 307570 T1376 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 307570 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 307571 T1376 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 307590 T1376 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 307592 T1376 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 307594 T1376 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 307596 T1376 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 307598 T1376 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 307601 T1376 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 307601 T1376 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 307602 T1376 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=8, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.83480000822237]
   [junit4]   2> 307602 T1376 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ce29336 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f2efe47),segFN=segments_1,generation=1}
   [junit4]   2> 307603 T1376 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 307603 T1376 oass.SolrIndexSearcher.<init> Opening Searcher@4cbb095e[collection1] main
   [junit4]   2> 307603 T1376 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 307604 T1376 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 307604 T1376 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 307605 T1376 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 307605 T1376 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 307605 T1376 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 307606 T1376 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 307606 T1376 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 307606 T1376 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 307607 T1376 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 307608 T1377 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4cbb095e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 307608 T1376 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 307609 T1380 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13220/_l/h collection:collection1 shard:shard1
   [junit4]   2> 307609 T1319 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 307610 T1319 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 307610 T1380 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 307614 T1380 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 307616 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 307616 T1380 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 307616 T1380 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C465 name=collection1 org.apache.solr.core.SolrCore@88fb02c url=http://127.0.0.1:13220/_l/h/collection1 node=127.0.0.1:13220__l%2Fh C465_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:13220/_l/h, node_name=127.0.0.1:13220__l%2Fh, core=collection1, state=down}
   [junit4]   2> 307617 T1380 C465 P13220 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13220/_l/h/collection1/
   [junit4]   2> 307617 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 307617 T1380 C465 P13220 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 307617 T1380 C465 P13220 oasc.SyncStrategy.syncToMe http://127.0.0.1:13220/_l/h/collection1/ has no replicas
   [junit4]   2> 307618 T1380 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13220/_l/h/collection1/ shard1
   [junit4]   2> 307618 T1380 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 307618 T1346 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 307619 T1375 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 307619 T1375 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 307622 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13220/_l/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 307623 T1346 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 307624 T1375 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 307624 T1375 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 307626 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 307672 T1380 oasc.ZkController.register We are http://127.0.0.1:13220/_l/h/collection1/ and leader is http://127.0.0.1:13220/_l/h/collection1/
   [junit4]   2> 307673 T1380 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13220/_l/h
   [junit4]   2> 307673 T1380 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 307673 T1380 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 307673 T1380 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 307675 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 307676 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13220/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13220__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 307677 T1346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13220/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:13220__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 307678 T1380 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 307678 T1346 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 307679 T1375 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 307679 T1375 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 308750 T1319 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 308752 T1319 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 308755 T1319 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13223
   [junit4]   2> 308756 T1319 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 308756 T1319 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 308756 T1319 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004
   [junit4]   2> 308756 T1319 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/'
   [junit4]   2> 308787 T1319 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/solr.xml
   [junit4]   2> 308857 T1319 oasc.CoreContainer.<init> New CoreContainer 328274594
   [junit4]   2> 308858 T1319 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/]
   [junit4]   2> 308859 T1319 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 308859 T1319 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 308859 T1319 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 308859 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 308860 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 308860 T1319 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 308860 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 308860 T1319 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 308861 T1319 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 308861 T1319 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 308861 T1319 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 308862 T1319 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 308862 T1319 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 308862 T1319 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 308863 T1319 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13193/solr
   [junit4]   2> 308863 T1319 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 308863 T1319 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 308864 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 308866 T1392 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64a03d94 name:ZooKeeperConnection Watcher:127.0.0.1:13193 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 308867 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 308867 T1319 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 308870 T1319 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 308872 T1395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6dc47481 name:ZooKeeperConnection Watcher:127.0.0.1:13193/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 308872 T1319 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 308877 T1319 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 309882 T1319 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13223__l%2Fh
   [junit4]   2> 309883 T1319 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13223__l%2Fh
   [junit4]   2> 309887 T1319 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 309895 T1396 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 309895 T1396 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 309897 T1345 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 309897 T1396 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 309898 T1396 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 309898 T1346 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13223/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13223__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 309898 T1396 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 309899 T1346 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13223/_l/h",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13223__l%2Fh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 309899 T1346 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 309899 T1346 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 309899 T1396 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 309900 T1396 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 309900 T1346 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 309901 T1395 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 309901 T1375 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json, has occurred - updating... 
   [junit4]   2> 309902 T1395 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 309902 T1375 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 310900 T1396 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 310901 T1396 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 310902 T1396 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 310902 T1396 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 310902 T1396 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 310903 T1396 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/collection1/'
   [junit4]   2> 310904 T1396 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 310905 T1396 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 310906 T1396 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 310939 T1396 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 310942 T1396 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 310977 T1396 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 311025 T1396 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 311127 T1396 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 311137 T1396 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 311520 T1396 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 311533 T1396 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 311536 T1396 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 311548 T1396 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 311552 T1396 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 311556 T1396 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 311557 T1396 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 311558 T1396 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 311559 T1396 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 311559 T1396 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 311560 T1396 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 311560 T1396 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 311560 T1396 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 311561 T1396 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-004/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty2/
   [junit4]   2> 311561 T1396 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@16552add
   [junit4]   2> 311562 T1396 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty2
   [junit4]   2> 311563 T1396 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty2/index/
   [junit4]   2> 311563 T1396 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 311563 T1396 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.AliasIntegrationTest-6A3703BC82601E40-001/tempDir-001/jetty2/index
   [junit4]   2> 311564 T1396 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=65.9404296875, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=11.134789867608193, segmentsPerTier=48.0, maxCFSSegmentSizeMB=0.9462890625, noCFSRatio=1.0
   [junit4]   2> 311565 T1396 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@61ab1ef8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@297e5de0),segFN=segments_1,generation=1}
   [junit4]   2> 311565 T1396 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 311570 T1396 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 311571 T1396 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 311571 T1396 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 311571 T1396 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 311572 T1396 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 311572 T1396 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 311573 T1396 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 311573 T1396 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 311573 T1396 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 311574 T1396 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 311574 T1396 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 311575 T1396 oasc.RequestHandlers.initHandlersFromConfig created /upd

[...truncated too long message...]

127.0.0.1:24507: Could not fully create collection: awholynewstresscollection_collection4_7
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3673, name=Thread-1449, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:24500, http://127.0.0.1:24524, http://127.0.0.1:24507, http://127.0.0.1:24518, http://127.0.0.1:24492]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6A3703BC82601E40]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:24500, http://127.0.0.1:24524, http://127.0.0.1:24507, http://127.0.0.1:24518, http://127.0.0.1:24492]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:24507: Could not fully create collection: awholynewstresscollection_collection5_8
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 moreThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3669, name=Thread-1445, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:24500, http://127.0.0.1:24524, http://127.0.0.1:24507, http://127.0.0.1:24518, http://127.0.0.1:24492]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6A3703BC82601E40]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:915)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:24500, http://127.0.0.1:24524, http://127.0.0.1:24507, http://127.0.0.1:24518, http://127.0.0.1:24492]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:332)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:905)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:24507: Could not fully create collection: awholynewstresscollection_collection1_8
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:558)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:353)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:312)
   [junit4]    > 	... 6 more
   [junit4]   2> 1987685 T3322 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest-6A3703BC82601E40-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=Lucene50(blocksize=128), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=PostingsFormat(name=LuceneFixedGap), a_i1=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), other_tl1=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=sq, timezone=America/Iqaluit
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=168234992,total=431489024
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedResourceStorage, TestClusterStateMutator, SolrCoreTest, TestManagedStopFilterFactory, HdfsRecoveryZkTest, SolrXmlInZkTest, TestLeaderElectionZkExpiry, RAMDirectoryFactoryTest, TestConfigReload, TestQueryTypes, TestInitParams, TestIndexSearcher, TestSchemaManager, TestJoin, BadIndexSchemaTest, BinaryUpdateRequestHandlerTest, ShardRoutingTest, TestRestManager, AddBlockUpdateTest, TestRawResponseWriter, TestManagedSynonymFilterFactory, CoreAdminHandlerTest, ClusterStateUpdateTest, SolrIndexConfigTest, TestInfoStreamLogging, XsltUpdateRequestHandlerTest, LeaderElectionTest, TestPivotHelperCode, TermVectorComponentDistributedTest, RemoteQueryErrorTest, TestManagedResource, AnalysisAfterCoreReloadTest, AssignTest, UUIDFieldTest, IndexSchemaRuntimeFieldTest, TestIntervalFaceting, HdfsCollectionsAPIDistributedZkTest, TestPostingsSolrHighlighter, CachingDirectoryFactoryTest, TestRangeQuery, URLClassifyProcessorTest, DocValuesTest, TestConfigOverlay, UnloadDistributedZkTest, OverseerStatusTest, SuggesterWFSTTest, TestSystemIdResolver, TestIndexingPerformance, TestArbitraryIndexDir, TestFieldTypeResource, TestUniqueKeyFieldResource, TestFiltering, TestNoOpRegenerator, TestXIncludeConfig, DocValuesMultiTest, BasicDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J3 in 1110.22s, 1 test, 1 error <<< FAILURES!

[...truncated 843 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 458 suites, 1859 tests, 2 errors, 57 ignored (10 assumptions)

Total time: 157 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 1271740603 bytes
Compression is 0.0%
Took 3 min 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 719 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/719/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Error from server at http://127.0.0.1:65196: java.lang.NullPointerException 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error from server at http://127.0.0.1:65196: java.lang.NullPointerException

	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:448)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:200)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:869)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10344 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/init-core-data-001
   [junit4]   2> 787888 T2610 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 787888 T2610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 787894 T2610 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 787895 T2610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 787897 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 787996 T2610 oasc.ZkTestServer.run start zk server on port:65188
   [junit4]   2> 787997 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 787998 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 788001 T2618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f481e54 name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 788001 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 788002 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 788002 T2610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 788005 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 788006 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 788007 T2621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ec995dc name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 788008 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 788008 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 788008 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 788010 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 788012 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 788013 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 788015 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 788016 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 788018 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 788019 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 788121 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 788122 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 788124 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 788124 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 788126 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 788127 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 788128 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 788129 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 788131 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 788131 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 788133 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 788134 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 788135 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 788136 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 788138 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 788138 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 788140 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 788141 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using new style solr.xml
   [junit4]   2> 789323 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 789326 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65191
   [junit4]   2> 789326 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 789326 T2610 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 789327 T2610 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002
   [junit4]   2> 789327 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/'
   [junit4]   2> 789356 T2610 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/solr.xml
   [junit4]   2> 789414 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002
   [junit4]   2> 789415 T2610 oasc.CoreContainer.<init> New CoreContainer 529131042
   [junit4]   2> 789415 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/]
   [junit4]   2> 789417 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 789417 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 789417 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 789418 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 789418 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 789418 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 789419 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 789419 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 789419 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 789420 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 789420 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 789421 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 789421 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 789421 T2610 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 789422 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65188/solr
   [junit4]   2> 789422 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 789422 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 789423 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 789425 T2633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f892f95 name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 789426 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 789426 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 789428 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 789429 T2636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14b8ccad name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 789429 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 789431 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 789433 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 789435 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 789437 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 789439 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 789441 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 789442 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65191_
   [junit4]   2> 789443 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65191_
   [junit4]   2> 789444 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 789446 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 789447 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 789448 T2610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:65191_
   [junit4]   2> 789449 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 789450 T2610 oasc.Overseer.start Overseer (id=93058879544688643-127.0.0.1:65191_-n_0000000000) starting
   [junit4]   2> 789452 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 789457 T2610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 789458 T2638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 789458 T2610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 789460 T2610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 789461 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 789463 T2637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 789465 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002
   [junit4]   2> 790403 T2610 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/conf/
   [junit4]   2> 790458 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 790459 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 790459 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 790459 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 790460 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 790462 T2643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e6be45e name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 790462 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 790462 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 790464 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 790466 T2610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 790466 T2610 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 790466 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 790467 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 790468 T2646 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1367c525 name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 790468 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 790468 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 790470 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 790472 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 0, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":2} current state version: 0
   [junit4]   2> 790472 T2637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 790473 T2637 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 790474 T2643 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 790474 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 791375 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 791376 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 791379 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65196
   [junit4]   2> 791379 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 791379 T2610 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 791379 T2610 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003
   [junit4]   2> 791380 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/'
   [junit4]   2> 791409 T2610 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/solr.xml
   [junit4]   2> 791450 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003
   [junit4]   2> 791451 T2610 oasc.CoreContainer.<init> New CoreContainer 1879648753
   [junit4]   2> 791451 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/]
   [junit4]   2> 791452 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 791452 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 791452 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 791452 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 791453 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 791453 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 791453 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 791453 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 791454 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 791454 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 791454 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 791455 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 791455 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 791455 T2610 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 791455 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65188/solr
   [junit4]   2> 791456 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 791456 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 791456 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 791458 T2658 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d054849 name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 791459 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 791459 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 791461 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 791463 T2661 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ad617e0 name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 791463 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 791467 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 792471 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65196_
   [junit4]   2> 792472 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65196_
   [junit4]   2> 792475 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 792479 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003
   [junit4]   2> 793495 T2610 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/conf/
   [junit4]   2> 793499 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 793500 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 793500 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 794412 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 794412 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 794415 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65199
   [junit4]   2> 794415 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 794415 T2610 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 794416 T2610 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004
   [junit4]   2> 794416 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/'
   [junit4]   2> 794445 T2610 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/solr.xml
   [junit4]   2> 794487 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004
   [junit4]   2> 794488 T2610 oasc.CoreContainer.<init> New CoreContainer 177085433
   [junit4]   2> 794488 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/]
   [junit4]   2> 794489 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 794489 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 794489 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 794490 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 794490 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 794490 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 794490 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 794490 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 794491 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 794491 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 794491 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 794492 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 794492 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 794492 T2610 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 794492 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65188/solr
   [junit4]   2> 794493 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 794493 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 794494 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 794495 T2674 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@396eb78f name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 794496 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 794496 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 794498 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 794499 T2677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4af50d08 name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 794499 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 794504 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 795509 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65199_
   [junit4]   2> 795510 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65199_
   [junit4]   2> 795513 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 795518 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004
   [junit4]   2> 796786 T2610 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/conf/
   [junit4]   2> 796794 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 796795 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 796795 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 797708 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 797708 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 797711 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65202
   [junit4]   2> 797711 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 797712 T2610 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 797712 T2610 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005
   [junit4]   2> 797712 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005/'
   [junit4]   2> 797741 T2610 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005/solr.xml
   [junit4]   2> 797783 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005
   [junit4]   2> 797784 T2610 oasc.CoreContainer.<init> New CoreContainer 1431532538
   [junit4]   2> 797784 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005/]
   [junit4]   2> 797785 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 797785 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 797785 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 797785 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 797786 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 797786 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 797786 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 797786 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 797787 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 797787 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 797787 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 797788 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 797788 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 797788 T2610 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 797788 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65188/solr
   [junit4]   2> 797789 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 797789 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 797789 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797791 T2690 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@289e4ee6 name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797792 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797792 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 797794 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797795 T2693 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2bafe6dc name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797796 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797799 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 798803 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65202_
   [junit4]   2> 798804 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65202_
   [junit4]   2> 798807 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 798812 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005
   [junit4]   2> 799702 T2610 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-005/conf/
   [junit4]   2> 799704 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 799705 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 799705 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 800624 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 800625 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 800627 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65205
   [junit4]   2> 800628 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 800628 T2610 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 800628 T2610 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006
   [junit4]   2> 800628 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/'
   [junit4]   2> 800658 T2610 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/solr.xml
   [junit4]   2> 800699 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006
   [junit4]   2> 800700 T2610 oasc.CoreContainer.<init> New CoreContainer 1298805694
   [junit4]   2> 800700 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/]
   [junit4]   2> 800701 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 800701 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 800701 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 800702 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 800702 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 800702 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 800702 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 800702 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 800703 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 800703 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 800703 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=false
   [junit4]   2> 800704 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 800704 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 800704 T2610 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 800704 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65188/solr
   [junit4]   2> 800705 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 800705 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 800706 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 800707 T2706 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73cbdb29 name:ZooKeeperConnection Watcher:127.0.0.1:65188 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 800708 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 800708 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 800710 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 800711 T2709 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61caf295 name:ZooKeeperConnection Watcher:127.0.0.1:65188/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 800711 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 800735 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 801740 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65205_
   [junit4]   2> 801741 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65205_
   [junit4]   2> 801745 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 801749 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006
   [junit4]   2> 803097 T2610 oasc.CorePropertiesLocator.discoverUnder Found core conf in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/conf/
   [junit4]   2> 803102 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 803103 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 803103 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 803106 T2650 oasha.CollectionsHandler.handleCreateAction Creating Collection : wt=javabin&name=nodes_used_collection&action=CREATE&replicationFactor=2&numShards=2&version=2
   [junit4]   2> 803107 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/collection-queue-work state SyncConnected
   [junit4]   2> 803108 T2638 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 803109 T2711 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : create , {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"}
   [junit4]   2> 803110 T2711 oasc.OverseerCollectionProcessor.createConfNode Only one config set found in zk - using it:conf1
   [junit4]   2> 803110 T2711 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/nodes_used_collection 
   [junit4]   2> 803111 T2711 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 803113 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 803114 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2"} current state version: 1
   [junit4]   2> 803114 T2637 oasco.ClusterStateMutator.createCollection building a new cName: nodes_used_collection
   [junit4]   2> 803115 T2709 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803116 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803115 T2677 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803115 T2643 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803115 T2693 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803115 T2661 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803314 T2711 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection nodes_used_collection, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 803315 T2711 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:65205_
   [junit4]   2> 803315 T2711 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:65196_
   [junit4]   2> 803317 T2711 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:65191_
   [junit4]   2> 803318 T2703 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=nodes_used_collection&collection.configName=conf1&wt=javabin&shard=shard1&name=nodes_used_collection_shard1_replica1&action=CREATE&numShards=2&version=2
   [junit4]   2> 803318 T2711 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:65199_
   [junit4]   2> 803318 T2649 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=nodes_used_collection&collection.configName=conf1&wt=javabin&shard=shard1&name=nodes_used_collection_shard1_replica2&action=CREATE&numShards=2&version=2
   [junit4]   2> 803319 T2626 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=nodes_used_collection&collection.configName=conf1&wt=javabin&shard=shard2&name=nodes_used_collection_shard2_replica1&action=CREATE&numShards=2&version=2
   [junit4]   2> 803320 T2665 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection=nodes_used_collection&collection.configName=conf1&wt=javabin&shard=shard2&name=nodes_used_collection_shard2_replica2&action=CREATE&numShards=2&version=2
   [junit4]   2> 803320 T2649 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 803320 T2703 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 803321 T2649 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 803322 T2703 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 803321 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 803322 T2626 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down collection=nodes_used_collection
   [junit4]   2> 803323 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65196",
   [junit4]   2> 	  "node_name":"127.0.0.1:65196_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2"} current state version: 2
   [junit4]   2> 803323 T2665 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down collection=nodes_used_collection
   [junit4]   2> 803323 T2626 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 803323 T2637 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65196",
   [junit4]   2> 	  "node_name":"127.0.0.1:65196_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2"}
   [junit4]   2> 803323 T2665 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 803325 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65205",
   [junit4]   2> 	  "node_name":"127.0.0.1:65205_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1"} current state version: 2
   [junit4]   2> 803326 T2637 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65205",
   [junit4]   2> 	  "node_name":"127.0.0.1:65205_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1"}
   [junit4]   2> 803327 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 803328 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 3, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65191",
   [junit4]   2> 	  "node_name":"127.0.0.1:65191_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1"} current state version: 2
   [junit4]   2> 803328 T2637 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65191",
   [junit4]   2> 	  "node_name":"127.0.0.1:65191_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1"}
   [junit4]   2> 803330 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65199",
   [junit4]   2> 	  "node_name":"127.0.0.1:65199_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2"} current state version: 2
   [junit4]   2> 803331 T2637 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65199",
   [junit4]   2> 	  "node_name":"127.0.0.1:65199_",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2"}
   [junit4]   2> 803332 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 803333 T2677 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803333 T2643 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803333 T2693 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803333 T2661 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803333 T2709 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 803333 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 804322 T2703 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 804322 T2649 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 804323 T2703 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 804323 T2649 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 804323 T2626 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 804324 T2649 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 804324 T2703 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 804325 T2649 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 804324 T2665 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 804324 T2626 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 804326 T2649 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 804326 T2626 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 804326 T2665 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 804325 T2703 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 804327 T2665 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 804327 T2626 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 804326 T2649 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 804328 T2703 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 804327 T2665 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 804329 T2703 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 804328 T2626 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 804329 T2665 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 804330 T2626 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 804330 T2665 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 804379 T2649 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 804381 T2703 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 804384 T2665 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 804389 T2626 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 804403 T2649 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 804404 T2703 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 804404 T2626 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 804404 T2665 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 804485 T2626 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 804485 T2665 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 804486 T2703 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 804486 T2649 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.0.0
   [junit4]   2> 804551 T2626 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 804552 T2665 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 804552 T2649 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 804553 T2626 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 804553 T2703 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 804563 T2626 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 804653 T2649 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 804653 T2665 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 804654 T2703 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 804665 T2649 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 804665 T2665 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 804665 T2703 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 804995 T2626 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 805007 T2626 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 805011 T2626 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 805030 T2626 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805035 T2626 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805040 T2626 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805041 T2626 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805043 T2626 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805043 T2626 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805044 T2626 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805045 T2626 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805045 T2626 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using configuration from collection nodes_used_collection
   [junit4]   2> 805045 T2626 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 805046 T2626 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 805046 T2626 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7d7eaf74
   [junit4]   2> 805047 T2626 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/data
   [junit4]   2> 805048 T2626 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 805048 T2626 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 805049 T2626 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-002/nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 805049 T2626 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=36.44921875, floorSegmentMB=2.0673828125, forceMergeDeletesPctAllowed=25.828742405928722, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43319506264927843
   [junit4]   2> 805050 T2626 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c6fdf6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4b4b2470),segFN=segments_1,generation=1}
   [junit4]   2> 805051 T2626 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 805056 T2626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 805057 T2626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 805057 T2626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 805057 T2626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 805058 T2626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 805058 T2626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 805059 T2626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 805059 T2626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 805060 T2626 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 805061 T2626 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 805061 T2626 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 805062 T2626 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 805062 T2626 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 805063 T2626 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 805063 T2626 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 805064 T2626 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 805065 T2626 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 805065 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 805066 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 805067 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 805068 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 805068 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 805069 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 805070 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 805070 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 805071 T2626 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 805072 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 805072 T2626 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 805096 T2626 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 805099 T2626 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 805102 T2626 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 805104 T2626 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 805107 T2626 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 805113 T2626 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 805114 T2626 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 805114 T2626 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=7, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.240234375, noCFSRatio=0.4601688779871316]
   [junit4]   2> 805115 T2626 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c6fdf6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4b4b2470),segFN=segments_1,generation=1}
   [junit4]   2> 805116 T2626 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 805116 T2626 oass.SolrIndexSearcher.<init> Opening Searcher@56c75526[nodes_used_collection_shard2_replica1] main
   [junit4]   2> 805116 T2626 oascc.ZkStateReader.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 805117 T2626 oascc.ZkStateReader.readConfigName path=/collections/nodes_used_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 805118 T2626 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 805118 T2626 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 805118 T2626 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 805119 T2626 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 805119 T2626 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 805120 T2626 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 805120 T2626 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 805121 T2626 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 805122 T2716 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@56c75526[nodes_used_collection_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 805122 T2626 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica1
   [junit4]   2> 805123 T2626 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica1 address:http://127.0.0.1:65191 collection:nodes_used_collection shard:shard2
   [junit4]   2> 805123 T2626 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 805124 T2649 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 805125 T2703 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 805125 T2665 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 805128 T2626 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 805130 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 805130 T2626 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=179999ms
   [junit4]   2> 805130 T2649 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 805131 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"nodes_used_collection"} current state version: 3
   [junit4]   2> 805132 T2703 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 805132 T2665 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 805132 T2709 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805133 T2643 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805133 T2693 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805132 T2677 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805132 T2661 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805132 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 805134 T2649 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 805136 T2703 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 805136 T2665 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 805155 T2665 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805155 T2649 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805159 T2703 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805160 T2649 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805160 T2665 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805164 T2703 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 805165 T2649 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805165 T2665 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805166 T2649 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805166 T2665 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805167 T2649 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805167 T2665 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805167 T2649 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805168 T2665 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805169 T2649 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805169 T2665 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805169 T2665 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805169 T2703 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805169 T2649 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805170 T2665 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using configuration from collection nodes_used_collection
   [junit4]   2> 805170 T2703 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805170 T2649 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using configuration from collection nodes_used_collection
   [junit4]   2> 805171 T2703 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805171 T2665 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 805172 T2703 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 805171 T2649 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 805172 T2665 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/nodes_used_collection_shard2_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 805173 T2703 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 805173 T2649 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/nodes_used_collection_shard1_replica2/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-003/nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 805174 T2703 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 805173 T2665 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7d7eaf74
   [junit4]   2> 805174 T2703 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using configuration from collection nodes_used_collection
   [junit4]   2> 805174 T2649 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7d7eaf74
   [junit4]   2> 805175 T2703 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 805176 T2665 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-004/nodes_used_collection_shard2_replica2/data
   [junit4]   2> 805176 T2703 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001/tempDir-006/nodes_used_collection_shard1_replica1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.

[...truncated too long message...]

lDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:904)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error from server at http://127.0.0.1:65202: Could not fully create collection: awholynewstresscollection_collection0_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
   [junit4]    > 	... 6 moreThrowable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2852, name=Thread-1213, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:65202, http://127.0.0.1:65196, http://127.0.0.1:65191, http://127.0.0.1:65205, http://127.0.0.1:65199]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DAB487F4ABE707AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:914)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:65202, http://127.0.0.1:65196, http://127.0.0.1:65191, http://127.0.0.1:65205, http://127.0.0.1:65199]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:904)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error from server at http://127.0.0.1:65202: Could not fully create collection: awholynewstresscollection_collection5_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
   [junit4]    > 	... 6 moreThrowable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2849, name=Thread-1210, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    > Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:65202, http://127.0.0.1:65196, http://127.0.0.1:65191, http://127.0.0.1:65205, http://127.0.0.1:65199]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DAB487F4ABE707AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:914)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:65202, http://127.0.0.1:65196, http://127.0.0.1:65191, http://127.0.0.1:65205, http://127.0.0.1:65199]
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.sendRequest(CloudSolrServer.java:1015)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.requestWithRetryOnStaleState(CloudSolrServer.java:793)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:736)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1640)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:904)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error from server at http://127.0.0.1:65202: Could not fully create collection: awholynewstresscollection_collection2_6
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:213)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:209)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:354)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:313)
   [junit4]    > 	... 6 more
   [junit4]   2> 1790878 T2610 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest-DAB487F4ABE707AE-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en, timezone=America/Virgin
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=77691416,total=496500736
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, BasicZkTest, HdfsWriteToMultipleCollectionsTest, TestPerFieldSimilarity, SortByFunctionTest, TestOmitPositions, TestFileDictionaryLookup, CoreAdminRequestStatusTest, HighlighterMaxOffsetTest, DOMUtilTest, DateRangeFieldTest, DateFieldTest, CoreMergeIndexesAdminHandlerTest, WordBreakSolrSpellCheckerTest, ZkStateWriterTest, BlockCacheTest, TestAnalyzeInfixSuggestions, ZkControllerTest, TestSolrDeletionPolicy1, DocumentAnalysisRequestHandlerTest, ResourceLoaderTest, TestSolrConfigHandlerCloud, SimplePostToolTest, DistributedTermsComponentTest, CoreContainerCoreInitFailuresTest, TestLRUStatsCache, LoggingHandlerTest, TestDefaultSearchFieldResource, TestDocBasedVersionConstraints, ChaosMonkeySafeLeaderTest, OpenExchangeRatesOrgProviderTest, IndexSchemaTest, TestRecoveryHdfs, PingRequestHandlerTest, FieldMutatingUpdateProcessorTest, ConvertedLegacyTest, DisMaxRequestHandlerTest, AutoCommitTest, ExternalFileFieldSortTest, SuggestComponentTest, TestBlendedInfixSuggestions, TestSolrDynamicMBean, TestSchemaResource, MinimalSchemaTest, TestQueryUtils, SolrInfoMBeanTest, CircularListTest, HdfsChaosMonkeySafeLeaderTest, FullSolrCloudDistribCmdsTest, HighlighterTest, TestBulkSchemaConcurrent, TestBM25SimilarityFactory, DistributedFacetPivotSmallTest, TestStressVersions, TestCustomSort, DocValuesMissingTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J1 in 1003.03s, 1 test, 1 error <<< FAILURES!

[...truncated 752 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:190: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1363: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:970: There were test failures: 458 suites, 1859 tests, 1 error, 57 ignored (10 assumptions)

Total time: 148 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 988127897 bytes
Compression is 0.0%
Took 2 min 50 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure